1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright (c) 2010, Intel Corporation.
26 * All rights reserved.
27 */
28
29 #include <sys/asm_linkage.h>
30 #include <sys/asm_misc.h>
31 #include <sys/regset.h>
32 #include <sys/privregs.h>
33 #include <sys/x86_archext.h>
34
35 #if !defined(__lint)
36 #include <sys/segments.h>
37 #include "assym.h"
38 #endif
39
40 /*
41 * Our assumptions:
42 * - We are running in real mode.
43 * - Interrupts are disabled.
44 * - Selectors are equal (cs == ds == ss) for all real mode code
45 * - The GDT, IDT, ktss and page directory has been built for us
46 *
47 * Our actions:
48 * Start CPU:
49 * - We start using our GDT by loading correct values in the
50 * selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
51 * gs=KGS_SEL).
52 * - We change over to using our IDT.
53 * - We load the default LDT into the hardware LDT register.
54 * - We load the default TSS into the hardware task register.
55 * - call mp_startup(void) indirectly through the T_PC
56 * Stop CPU:
57 * - Put CPU into halted state with interrupts disabled
58 *
59 */
60
61 #if defined(__lint)
62
63 void
64 real_mode_start_cpu(void)
65 {}
66
67 void
68 real_mode_stop_cpu_stage1(void)
69 {}
70
71 void
72 real_mode_stop_cpu_stage2(void)
73 {}
74
75 #else /* __lint */
76
77 #if defined(__amd64)
78
79 ENTRY_NP(real_mode_start_cpu)
80
81 /*
82 * NOTE: The GNU assembler automatically does the right thing to
83 * generate data size operand prefixes based on the code size
84 * generation mode (e.g. .code16, .code32, .code64) and as such
85 * prefixes need not be used on instructions EXCEPT in the case
86 * of address prefixes for code for which the reference is not
87 * automatically of the default operand size.
88 */
89 .code16
90 cli
91 movw %cs, %ax
92 movw %ax, %ds /* load cs into ds */
93 movw %ax, %ss /* and into ss */
94
95 /*
96 * Helps in debugging by giving us the fault address.
97 *
98 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
99 */
100 movl $0xffc, %esp
101 movl %cr0, %eax
102
103 /*
104 * Enable protected-mode, write protect, and alignment mask
105 */
106 orl $(CR0_PE|CR0_WP|CR0_AM), %eax
107 movl %eax, %cr0
108
109 /*
110 * Do a jmp immediately after writing to cr0 when enabling protected
111 * mode to clear the real mode prefetch queue (per Intel's docs)
112 */
113 jmp pestart
114
115 pestart:
116 /*
117 * 16-bit protected mode is now active, so prepare to turn on long
118 * mode.
119 *
120 * Note that we currently assume that if we're attempting to run a
121 * kernel compiled with (__amd64) #defined, the target CPU has long
122 * mode support.
123 */
124
125 #if 0
126 /*
127 * If there's a chance this might not be true, the following test should
128 * be done, with the no_long_mode branch then doing something
129 * appropriate:
130 */
131
132 movl $0x80000000, %eax /* get largest extended CPUID */
133 cpuid
134 cmpl $0x80000000, %eax /* check if > 0x80000000 */
135 jbe no_long_mode /* nope, no long mode */
136 movl $0x80000001, %eax
137 cpuid /* get extended feature flags */
138 btl $29, %edx /* check for long mode */
139 jnc no_long_mode /* long mode not supported */
140 #endif
141
142 /*
143 * Add any initial cr4 bits
144 */
145 movl %cr4, %eax
146 addr32 orl CR4OFF, %eax
147
148 /*
149 * Enable PAE mode (CR4.PAE)
150 */
151 orl $CR4_PAE, %eax
152 movl %eax, %cr4
153
154 /*
155 * Point cr3 to the 64-bit long mode page tables.
156 *
157 * Note that these MUST exist in 32-bit space, as we don't have
158 * a way to load %cr3 with a 64-bit base address for the page tables
159 * until the CPU is actually executing in 64-bit long mode.
160 */
161 addr32 movl CR3OFF, %eax
162 movl %eax, %cr3
163
164 /*
165 * Set long mode enable in EFER (EFER.LME = 1)
166 */
167 movl $MSR_AMD_EFER, %ecx
168 rdmsr
169 orl $AMD_EFER_LME, %eax
170 wrmsr
171
172 /*
173 * Finally, turn on paging (CR0.PG = 1) to activate long mode.
174 */
175 movl %cr0, %eax
176 orl $CR0_PG, %eax
177 movl %eax, %cr0
178
179 /*
180 * The instruction after enabling paging in CR0 MUST be a branch.
181 */
182 jmp long_mode_active
183
184 long_mode_active:
185 /*
186 * Long mode is now active but since we're still running with the
187 * original 16-bit CS we're actually in 16-bit compatability mode.
188 *
189 * We have to load an intermediate GDT and IDT here that we know are
190 * in 32-bit space before we can use the kernel's GDT and IDT, which
191 * may be in the 64-bit address space, and since we're in compatability
192 * mode, we only have access to 16 and 32-bit instructions at the
193 * moment.
194 */
195 addr32 lgdtl TEMPGDTOFF /* load temporary GDT */
196 addr32 lidtl TEMPIDTOFF /* load temporary IDT */
197
198 /*
199 * Do a far transfer to 64-bit mode. Set the CS selector to a 64-bit
200 * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump
201 * to the real mode platter address of long_mode 64 as until the 64-bit
202 * CS is in place we don't have access to 64-bit instructions and thus
203 * can't reference a 64-bit %rip.
204 */
205 pushl $TEMP_CS64_SEL
206 addr32 pushl LM64OFF
207 lretl
208
209 .globl long_mode_64
210 long_mode_64:
211 .code64
212 /*
213 * We are now running in long mode with a 64-bit CS (EFER.LMA=1,
214 * CS.L=1) so we now have access to 64-bit instructions.
215 *
216 * First, set the 64-bit GDT base.
217 */
218 .globl rm_platter_pa
219 movl rm_platter_pa, %eax
220 lgdtq GDTROFF(%rax) /* load 64-bit GDT */
221
222 /*
223 * Save the CPU number in %r11; get the value here since it's saved in
224 * the real mode platter.
225 */
226 movl CPUNOFF(%rax), %r11d
227
228 /*
229 * Add rm_platter_pa to %rsp to point it to the same location as seen
230 * from 64-bit mode.
231 */
232 addq %rax, %rsp
233
234 /*
235 * Now do an lretq to load CS with the appropriate selector for the
236 * kernel's 64-bit GDT and to start executing 64-bit setup code at the
237 * virtual address where boot originally loaded this code rather than
238 * the copy in the real mode platter's rm_code array as we've been
239 * doing so far.
240 */
241 pushq $KCS_SEL
242 pushq $kernel_cs_code
243 lretq
244 .globl real_mode_start_cpu_end
245 real_mode_start_cpu_end:
246 nop
247
248 kernel_cs_code:
249 /*
250 * Complete the balance of the setup we need to before executing
251 * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS).
252 */
253 .globl rm_platter_va
254 movq rm_platter_va, %rax
255 lidtq IDTROFF(%rax)
256
257 movw $KDS_SEL, %ax
258 movw %ax, %ds
259 movw %ax, %es
260 movw %ax, %ss
261
262 movw $KTSS_SEL, %ax /* setup kernel TSS */
263 ltr %ax
264
265 xorw %ax, %ax /* clear LDTR */
266 lldt %ax
267
268 /*
269 * Set GS to the address of the per-cpu structure as contained in
270 * cpu[cpu_number].
271 *
272 * Unfortunately there's no way to set the 64-bit gsbase with a mov,
273 * so we have to stuff the low 32 bits in %eax and the high 32 bits in
274 * %edx, then call wrmsr.
275 */
276 leaq cpu(%rip), %rdi
277 movl (%rdi, %r11, 8), %eax
278 movl 4(%rdi, %r11, 8), %edx
279 movl $MSR_AMD_GSBASE, %ecx
280 wrmsr
281
282 /*
283 * Init FS and KernelGSBase.
284 *
285 * Based on code in mlsetup(), set them both to 8G (which shouldn't be
286 * valid until some 64-bit processes run); this will then cause an
287 * exception in any code that tries to index off them before they are
288 * properly setup.
289 */
290 xorl %eax, %eax /* low 32 bits = 0 */
291 movl $2, %edx /* high 32 bits = 2 */
292 movl $MSR_AMD_FSBASE, %ecx
293 wrmsr
294
295 movl $MSR_AMD_KGSBASE, %ecx
296 wrmsr
297
298 /*
299 * Init %rsp to the exception stack set in tss_ist1 and create a legal
300 * AMD64 ABI stack frame
301 */
302 movq %gs:CPU_TSS, %rax
303 movq TSS_IST1(%rax), %rsp
304 pushq $0 /* null return address */
305 pushq $0 /* null frame pointer terminates stack trace */
306 movq %rsp, %rbp /* stack aligned on 16-byte boundary */
307
308 movq %cr0, %rax
309 andq $~(CR0_TS|CR0_EM), %rax /* clear emulate math chip bit */
310 orq $(CR0_MP|CR0_NE), %rax
311 movq %rax, %cr0 /* set machine status word */
312
313 /*
314 * Before going any further, enable usage of page table NX bit if
315 * that's how our page tables are set up.
316 */
317 bt $X86FSET_NX, x86_featureset(%rip)
318 jnc 1f
319 movl $MSR_AMD_EFER, %ecx
320 rdmsr
321 orl $AMD_EFER_NXE, %eax
322 wrmsr
323 1:
324
325 /*
326 * Complete the rest of the setup and call mp_startup().
327 */
328 movq %gs:CPU_THREAD, %rax /* get thread ptr */
329 call *T_PC(%rax) /* call mp_startup */
330 /* not reached */
331 int $20 /* whoops, returned somehow! */
332
333 SET_SIZE(real_mode_start_cpu)
334
335 #elif defined(__i386)
336
337 ENTRY_NP(real_mode_start_cpu)
338
339 #if !defined(__GNUC_AS__)
340
341 cli
342 D16 movw %cs, %eax
343 movw %eax, %ds /* load cs into ds */
344 movw %eax, %ss /* and into ss */
345
346 /*
347 * Helps in debugging by giving us the fault address.
348 *
349 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
350 */
351 D16 movl $0xffc, %esp
352
353 D16 A16 lgdt %cs:GDTROFF
354 D16 A16 lidt %cs:IDTROFF
355 D16 A16 movl %cs:CR4OFF, %eax /* set up CR4, if desired */
356 D16 andl %eax, %eax
357 D16 A16 je no_cr4
358
359 D16 movl %eax, %ecx
360 D16 movl %cr4, %eax
361 D16 orl %ecx, %eax
362 D16 movl %eax, %cr4
363 no_cr4:
364 D16 A16 movl %cs:CR3OFF, %eax
365 A16 movl %eax, %cr3
366 movl %cr0, %eax
367
368 /*
369 * Enable protected-mode, paging, write protect, and alignment mask
370 */
371 D16 orl $[CR0_PG|CR0_PE|CR0_WP|CR0_AM], %eax
372 movl %eax, %cr0
373 jmp pestart
374
375 pestart:
376 D16 pushl $KCS_SEL
377 D16 pushl $kernel_cs_code
378 D16 lret
379 .globl real_mode_start_cpu_end
380 real_mode_start_cpu_end:
381 nop
382
383 .globl kernel_cs_code
384 kernel_cs_code:
385 /*
386 * At this point we are with kernel's cs and proper eip.
387 *
388 * We will be executing not from the copy in real mode platter,
389 * but from the original code where boot loaded us.
390 *
391 * By this time GDT and IDT are loaded as is cr3.
392 */
393 movw $KFS_SEL,%eax
394 movw %eax,%fs
395 movw $KGS_SEL,%eax
396 movw %eax,%gs
397 movw $KDS_SEL,%eax
398 movw %eax,%ds
399 movw %eax,%es
400 movl %gs:CPU_TSS,%esi
401 movw %eax,%ss
402 movl TSS_ESP0(%esi),%esp
403 movw $KTSS_SEL,%ax
404 ltr %ax
405 xorw %ax, %ax /* clear LDTR */
406 lldt %ax
407 movl %cr0,%edx
408 andl $-1![CR0_TS|CR0_EM],%edx /* clear emulate math chip bit */
409 orl $[CR0_MP|CR0_NE],%edx
410 movl %edx,%cr0 /* set machine status word */
411
412 /*
413 * Before going any further, enable usage of page table NX bit if
414 * that's how our page tables are set up.
415 */
416 bt $X86FSET_NX, x86_featureset
417 jnc 1f
418 movl %cr4, %ecx
419 andl $CR4_PAE, %ecx
420 jz 1f
421 movl $MSR_AMD_EFER, %ecx
422 rdmsr
423 orl $AMD_EFER_NXE, %eax
424 wrmsr
425 1:
426 movl %gs:CPU_THREAD, %eax /* get thread ptr */
427 call *T_PC(%eax) /* call mp_startup */
428 /* not reached */
429 int $20 /* whoops, returned somehow! */
430
431 #else
432
433 cli
434 mov %cs, %ax
435 mov %eax, %ds /* load cs into ds */
436 mov %eax, %ss /* and into ss */
437
438 /*
439 * Helps in debugging by giving us the fault address.
440 *
441 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
442 */
443 D16 mov $0xffc, %esp
444
445 D16 A16 lgdtl %cs:GDTROFF
446 D16 A16 lidtl %cs:IDTROFF
447 D16 A16 mov %cs:CR4OFF, %eax /* set up CR4, if desired */
448 D16 and %eax, %eax
449 D16 A16 je no_cr4
450
451 D16 mov %eax, %ecx
452 D16 mov %cr4, %eax
453 D16 or %ecx, %eax
454 D16 mov %eax, %cr4
455 no_cr4:
456 D16 A16 mov %cs:CR3OFF, %eax
457 A16 mov %eax, %cr3
458 mov %cr0, %eax
459
460 /*
461 * Enable protected-mode, paging, write protect, and alignment mask
462 */
463 D16 or $(CR0_PG|CR0_PE|CR0_WP|CR0_AM), %eax
464 mov %eax, %cr0
465 jmp pestart
466
467 pestart:
468 D16 pushl $KCS_SEL
469 D16 pushl $kernel_cs_code
470 D16 lret
471 .globl real_mode_start_cpu_end
472 real_mode_start_cpu_end:
473 nop
474 .globl kernel_cs_code
475 kernel_cs_code:
476 /*
477 * At this point we are with kernel's cs and proper eip.
478 *
479 * We will be executing not from the copy in real mode platter,
480 * but from the original code where boot loaded us.
481 *
482 * By this time GDT and IDT are loaded as is cr3.
483 */
484 mov $KFS_SEL, %ax
485 mov %eax, %fs
486 mov $KGS_SEL, %ax
487 mov %eax, %gs
488 mov $KDS_SEL, %ax
489 mov %eax, %ds
490 mov %eax, %es
491 mov %gs:CPU_TSS, %esi
492 mov %eax, %ss
493 mov TSS_ESP0(%esi), %esp
494 mov $(KTSS_SEL), %ax
495 ltr %ax
496 xorw %ax, %ax /* clear LDTR */
497 lldt %ax
498 mov %cr0, %edx
499 and $~(CR0_TS|CR0_EM), %edx /* clear emulate math chip bit */
500 or $(CR0_MP|CR0_NE), %edx
501 mov %edx, %cr0 /* set machine status word */
502
503 /*
504 * Before going any farther, enable usage of page table NX bit if
505 * that's how our page tables are set up.
506 */
507 bt $X86FSET_NX, x86_featureset
508 jnc 1f
509 movl %cr4, %ecx
510 andl $CR4_PAE, %ecx
511 jz 1f
512 movl $MSR_AMD_EFER, %ecx
513 rdmsr
514 orl $AMD_EFER_NXE, %eax
515 wrmsr
516 1:
517 mov %gs:CPU_THREAD, %eax /* get thread ptr */
518 call *T_PC(%eax) /* call mp_startup */
519 /* not reached */
520 int $20 /* whoops, returned somehow! */
521 #endif
522
523 SET_SIZE(real_mode_start_cpu)
524
525 #endif /* __amd64 */
526
527 #if defined(__amd64)
528
529 ENTRY_NP(real_mode_stop_cpu_stage1)
530
531 #if !defined(__GNUC_AS__)
532
533 /*
534 * For vulcan as we need to do a .code32 and mentally invert the
535 * meaning of the addr16 and data16 prefixes to get 32-bit access when
536 * generating code to be executed in 16-bit mode (sigh...)
537 */
538 .code32
539 cli
540 movw %cs, %ax
541 movw %ax, %ds /* load cs into ds */
542 movw %ax, %ss /* and into ss */
543
544 /*
545 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
546 */
547 movw $CPUHALTCODEOFF, %ax
548 .byte 0xff, 0xe0 /* jmp *%ax */
549
550 #else /* __GNUC_AS__ */
551
552 /*
553 * NOTE: The GNU assembler automatically does the right thing to
554 * generate data size operand prefixes based on the code size
555 * generation mode (e.g. .code16, .code32, .code64) and as such
556 * prefixes need not be used on instructions EXCEPT in the case
557 * of address prefixes for code for which the reference is not
558 * automatically of the default operand size.
559 */
560 .code16
561 cli
562 movw %cs, %ax
563 movw %ax, %ds /* load cs into ds */
564 movw %ax, %ss /* and into ss */
565
566 /*
567 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
568 */
569 movw $CPUHALTCODEOFF, %ax
570 jmp *%ax
571
572 #endif /* !__GNUC_AS__ */
573
574 .globl real_mode_stop_cpu_stage1_end
575 real_mode_stop_cpu_stage1_end:
576 nop
577
578 SET_SIZE(real_mode_stop_cpu_stage1)
579
580 #elif defined(__i386)
581
582 ENTRY_NP(real_mode_stop_cpu_stage1)
583
584 #if !defined(__GNUC_AS__)
585
586 cli
587 D16 movw %cs, %eax
588 movw %eax, %ds /* load cs into ds */
589 movw %eax, %ss /* and into ss */
590
591 /*
592 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
593 */
594 movw $CPUHALTCODEOFF, %ax
595 .byte 0xff, 0xe0 /* jmp *%ax */
596
597 #else /* __GNUC_AS__ */
598
599 cli
600 mov %cs, %ax
601 mov %eax, %ds /* load cs into ds */
602 mov %eax, %ss /* and into ss */
603
604 /*
605 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
606 */
607 movw $CPUHALTCODEOFF, %ax
608 jmp *%ax
609
610 #endif /* !__GNUC_AS__ */
611
612 .globl real_mode_stop_cpu_stage1_end
613 real_mode_stop_cpu_stage1_end:
614 nop
615
616 SET_SIZE(real_mode_stop_cpu_stage1)
617
618 #endif /* __amd64 */
619
620 ENTRY_NP(real_mode_stop_cpu_stage2)
621
622 movw $0xdead, %ax
623 movw %ax, CPUHALTEDOFF
624
625 real_mode_stop_cpu_loop:
626 /*
627 * Put CPU into halted state.
628 * Only INIT, SMI, NMI could break the loop.
629 */
630 hlt
631 jmp real_mode_stop_cpu_loop
632
633 .globl real_mode_stop_cpu_stage2_end
634 real_mode_stop_cpu_stage2_end:
635 nop
636
637 SET_SIZE(real_mode_stop_cpu_stage2)
638
639 #endif /* __lint */