26 * Copyright 2019 Joyent, Inc.
27 */
28
29 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
31 /* All Rights Reserved */
32
33 /* Copyright (c) 1987, 1988 Microsoft Corporation */
34 /* All Rights Reserved */
35
36
37 #include <sys/asm_linkage.h>
38 #include <sys/asm_misc.h>
39 #include <sys/regset.h>
40 #include <sys/privregs.h>
41 #include <sys/psw.h>
42 #include <sys/reboot.h>
43 #include <sys/x86_archext.h>
44 #include <sys/machparam.h>
45
46 #if defined(__lint)
47
48 #include <sys/types.h>
49 #include <sys/thread.h>
50 #include <sys/systm.h>
51 #include <sys/lgrp.h>
52 #include <sys/regset.h>
53 #include <sys/link.h>
54 #include <sys/bootconf.h>
55 #include <sys/bootsvcs.h>
56
57 #else /* __lint */
58
59 #include <sys/segments.h>
60 #include <sys/pcb.h>
61 #include <sys/trap.h>
62 #include <sys/ftrace.h>
63 #include <sys/traptrace.h>
64 #include <sys/clock.h>
65 #include <sys/cmn_err.h>
66 #include <sys/pit.h>
67 #include <sys/panic.h>
68
69 #if defined(__xpv)
70 #include <sys/hypervisor.h>
71 #endif
72
73 #include "assym.h"
74
75 /*
76 * Our assumptions:
77 * - We are running in protected-paged mode.
78 * - Interrupts are disabled.
116 .globl panic
117 .globl t0stack
118 .globl t0
119 .globl sysp
120 .globl edata
121
122 /*
123 * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
124 */
125 .globl bootops
126 .globl bootopsp
127
128 /*
129 * NOTE: t0stack should be the first thing in the data section so that
130 * if it ever overflows, it will fault on the last kernel text page.
131 */
132 .data
133 .comm t0stack, DEFAULTSTKSZ, 32
134 .comm t0, 4094, 32
135
136 #endif /* __lint */
137
138
139 #if defined(__amd64)
140
141 #if defined(__lint)
142
143 /* ARGSUSED */
144 void
145 _locore_start(struct boot_syscalls *sysp, ulong_t rsi, struct bootops *bop)
146 {}
147
148 #else /* __lint */
149
150 /*
151 * kobj_init() vectors us back to here with (note) a slightly different
152 * set of arguments than _start is given (see lint prototypes above).
153 *
154 * XXX Make this less vile, please.
155 */
156 ENTRY_NP(_locore_start)
157
158 /*
159 * %rdi = boot services (should die someday)
160 * %rdx = bootops
161 * end
162 */
163
164 leaq edata(%rip), %rbp /* reference edata for ksyms */
165 movq $0, (%rbp) /* limit stack back trace */
166
167 /*
168 * Initialize our stack pointer to the thread 0 stack (t0stack)
169 * and leave room for a "struct regs" for lwp0. Note that the
212 */
213 bts $X86FSET_CPUID, x86_featureset(%rip)
214
215 /*
216 * mlsetup() gets called with a struct regs as argument, while
217 * main takes no args and should never return.
218 */
219 xorl %ebp, %ebp
220 movq %rsp, %rdi
221 pushq %rbp
222 /* (stack pointer now aligned on 16-byte boundary right here) */
223 movq %rsp, %rbp
224 call mlsetup
225 call main
226 /* NOTREACHED */
227 leaq __return_from_main(%rip), %rdi
228 xorl %eax, %eax
229 call panic
230 SET_SIZE(_locore_start)
231
232 #endif /* __amd64 */
233 #endif /* __lint */
234
235 #if !defined(__lint)
236
237 __return_from_main:
238 .string "main() returned"
239 __unsupported_cpu:
240 .string "486 style cpu detected - no longer supported!"
241
242 #if defined(DEBUG)
243 _no_pending_updates:
244 .string "locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
245 #endif
246
247 #endif /* !__lint */
248
249 #if !defined(__amd64)
250
251 #if defined(__lint)
252
253 /* ARGSUSED */
254 void
255 _locore_start(struct boot_syscalls *sysp, struct bootops *bop)
256 {}
257
258 #else /* __lint */
259
260 /*
261 * kobj_init() vectors us back to here with (note) a slightly different
262 * set of arguments than _start is given (see lint prototypes above).
263 *
264 * XXX Make this less vile, please.
265 */
266 ENTRY_NP(_locore_start)
267
268 /*
269 * %ecx = boot services (should die someday)
270 * %ebx = bootops
271 */
272 mov $edata, %ebp / edata needs to be defined for ksyms
273 movl $0, (%ebp) / limit stack back trace
274
275 /*
276 * Initialize our stack pointer to the thread 0 stack (t0stack)
277 * and leave room for a phony "struct regs".
278 */
279 movl $t0stack + DEFAULTSTKSZ - REGSIZE, %esp
280
281 /*
282 * Save call back for special x86 boot services vector
283 */
284 mov %ecx, sysp / save call back for boot services
285
286 mov %ebx, bootops / save bootops
287 movl $bootops, bootopsp
288
289
290 /*
291 * Save all registers and flags
292 */
293 pushal
294 pushfl
295
296 #if !defined(__xpv)
297 /*
298 * Override bios settings and enable write protect and
299 * alignment check faults.
300 */
301 movl %cr0, %eax
302
303 /*
304 * enable WP for detecting faults, and enable alignment checking.
305 */
306 orl $_CONST(CR0_WP|CR0_AM), %eax
307 andl $_BITNOT(CR0_WT|CR0_CE), %eax
308 movl %eax, %cr0 / set the cr0 register correctly and
309 / override the BIOS setup
310
311 /*
312 * If bit 21 of eflags can be flipped, then cpuid is present
313 * and enabled.
314 */
315 pushfl
316 popl %ecx
317 movl %ecx, %eax
318 xorl $PS_ID, %eax / try complemented bit
319 pushl %eax
320 popfl
321 pushfl
322 popl %eax
323 cmpl %eax, %ecx
324 jne have_cpuid
325
326 /*
327 * cpuid may be disabled on Cyrix, try to detect Cyrix by the 5/2 test
328 * div does not modify the cc flags on Cyrix, even though this may
329 * also be true for other vendors, this is generally true only for
330 * newer models from those vendors that support and do not disable
331 * cpuid (usually because cpuid cannot be disabled)
332 */
333
334 /*
335 * clear cc flags
336 */
337 xorb %ah, %ah
338 sahf
339
340 /*
341 * perform 5/2 test
342 */
343 movw $5, %ax
344 movb $2, %bl
345 divb %bl
346
347 lahf
348 cmpb $2, %ah
349 jne cpu_486
350
351 /*
352 * div did not modify the cc flags, chances are the vendor is Cyrix
353 * assume the vendor is Cyrix and use the CCR's to enable cpuid
354 */
355 .set CYRIX_CRI, 0x22 / CR Index Register
356 .set CYRIX_CRD, 0x23 / CR Data Register
357
358 .set CYRIX_CCR3, 0xc3 / Config Control Reg 3
359 .set CYRIX_CCR4, 0xe8 / Config Control Reg 4
360 .set CYRIX_DIR0, 0xfe / Device Identification Reg 0
361 .set CYRIX_DIR1, 0xff / Device Identification Reg 1
362
363 /*
364 * even if the cpu vendor is Cyrix and the motherboard/chipset
365 * vendor decided to ignore lines A1-A4 for I/O addresses, I/O port
366 * 0x21 corresponds with 0x23 and since 0x22 is still untouched,
367 * the reads and writes of 0x21 are guaranteed to be off-chip of
368 * the cpu
369 */
370
371 /*
372 * enable read of ISR at I/O port 0x20
373 */
374 movb $0xb, %al
375 outb $MCMD_PORT
376
377 /*
378 * read IMR and store in %bl
379 */
380 inb $MIMR_PORT
381 movb %al, %bl
382
383 /*
384 * mask out all interrupts so that ISR will not change
385 */
386 movb $0xff, %al
387 outb $MIMR_PORT
388
389 /*
390 * reads of I/O port 0x22 on Cyrix are always directed off-chip
391 * make use of I/O pull-up to test for an unknown device on 0x22
392 */
393 inb $CYRIX_CRI
394 cmpb $0xff, %al
395 je port_22_free
396
397 /*
398 * motherboard/chipset vendor may be ignoring line A1 of I/O address
399 */
400 movb %al, %cl
401
402 /*
403 * if the ISR and the value read from 0x22 do not match then we have
404 * detected some unknown device, probably a chipset, at 0x22
405 */
406 inb $MCMD_PORT
407 cmpb %al, %cl
408 jne restore_IMR
409
410 port_22_free:
411 /*
412 * now test to see if some unknown device is using I/O port 0x23
413 *
414 * read the external I/O port at 0x23
415 */
416 inb $CYRIX_CRD
417
418 /*
419 * Test for pull-up at 0x23 or if I/O address line A1 is being ignored.
420 * IMR is 0xff so both tests are performed simultaneously.
421 */
422 cmpb $0xff, %al
423 jne restore_IMR
424
425 /*
426 * We are a Cyrix part. In case we are some model of Cx486 or a Cx586,
427 * record the type and fix it later if not.
428 */
429 movl $X86_VENDOR_Cyrix, x86_vendor
430 movl $X86_TYPE_CYRIX_486, x86_type
431
432 /*
433 * Try to read CCR3. All Cyrix cpu's which support cpuid have CCR3.
434 *
435 * load CCR3 index into CCR index register
436 */
437
438 movb $CYRIX_CCR3, %al
439 outb $CYRIX_CRI
440
441 /*
442 * If we are not a Cyrix cpu, then we have performed an external I/O
443 * cycle. If the CCR index was not valid for this Cyrix model, we may
444 * have performed an external I/O cycle as well. In these cases and
445 * if the motherboard/chipset vendor ignores I/O address line A1,
446 * then the PIC will have IRQ3 set at the lowest priority as a side
447 * effect of the above outb. We are reasonalbly confident that there
448 * is not an unknown device on I/O port 0x22, so there should have been
449 * no unpredictable side-effect of the above outb.
450 */
451
452 /*
453 * read CCR3
454 */
455 inb $CYRIX_CRD
456
457 /*
458 * If we are not a Cyrix cpu the inb above produced an external I/O
459 * cycle. If we are a Cyrix model that does not support CCR3 wex
460 * produced an external I/O cycle. In all known Cyrix models 6x86 and
461 * above, bit 3 of CCR3 is reserved and cannot be set to 1. In all
462 * Cyrix models prior to the 6x86 that supported CCR3, bits 4-7 are
463 * reserved as well. It is highly unlikely that CCR3 contains the value
464 * 0xff. We test to see if I/O port 0x23 is pull-up or the IMR and
465 * deduce we are not a Cyrix with support for cpuid if so.
466 */
467 cmpb $0xff, %al
468 je restore_PIC
469
470 /*
471 * There exist 486 ISA Cyrix chips that support CCR3 but do not support
472 * DIR0 and DIR1. If we try to read DIR0, we may generate external I/O
473 * cycles, the exact behavior is model specific and undocumented.
474 * Unfortunately these external I/O cycles may confuse some PIC's beyond
475 * recovery. Fortunatetly we can use the following undocumented trick:
476 * if bit 4 of CCR3 can be toggled, then DIR0 and DIR1 are supported.
477 * Pleasantly MAPEN contains bit 4 of CCR3, so this trick is guaranteed
478 * to work on all Cyrix cpu's which support cpuid.
479 */
480 movb %al, %dl
481 xorb $0x10, %dl
482 movb %al, %cl
483
484 /*
485 * write back CRR3 with toggled bit 4 to CCR3
486 */
487 movb $CYRIX_CCR3, %al
488 outb $CYRIX_CRI
489
490 movb %dl, %al
491 outb $CYRIX_CRD
492
493 /*
494 * read CCR3
495 */
496 movb $CYRIX_CCR3, %al
497 outb $CYRIX_CRI
498 inb $CYRIX_CRD
499 movb %al, %dl
500
501 /*
502 * restore CCR3
503 */
504 movb $CYRIX_CCR3, %al
505 outb $CYRIX_CRI
506
507 movb %cl, %al
508 outb $CYRIX_CRD
509
510 /*
511 * if bit 4 was not toggled DIR0 and DIR1 are not supported in which
512 * case we do not have cpuid anyway
513 */
514 andb $0x10, %al
515 andb $0x10, %dl
516 cmpb %al, %dl
517 je restore_PIC
518
519 /*
520 * read DIR0
521 */
522 movb $CYRIX_DIR0, %al
523 outb $CYRIX_CRI
524 inb $CYRIX_CRD
525
526 /*
527 * test for pull-up
528 */
529 cmpb $0xff, %al
530 je restore_PIC
531
532 /*
533 * Values of 0x20-0x27 in DIR0 are currently reserved by Cyrix for
534 * future use. If Cyrix ever produces a cpu that supports cpuid with
535 * these ids, the following test will have to change. For now we remain
536 * pessimistic since the formats of the CRR's may be different then.
537 *
538 * test for at least a 6x86, to see if we support both MAPEN and CPUID
539 */
540 cmpb $0x30, %al
541 jb restore_IMR
542
543 /*
544 * enable MAPEN
545 */
546 movb $CYRIX_CCR3, %al
547 outb $CYRIX_CRI
548
549 andb $0xf, %cl
550 movb %cl, %al
551 orb $0x10, %al
552 outb $CYRIX_CRD
553
554 /*
555 * select CCR4
556 */
557 movb $CYRIX_CCR4, %al
558 outb $CYRIX_CRI
559
560 /*
561 * read CCR4
562 */
563 inb $CYRIX_CRD
564
565 /*
566 * enable cpuid
567 */
568 orb $0x80, %al
569 movb %al, %dl
570
571 /*
572 * select CCR4
573 */
574 movb $CYRIX_CCR4, %al
575 outb $CYRIX_CRI
576
577 /*
578 * write CCR4
579 */
580 movb %dl, %al
581 outb $CYRIX_CRD
582
583 /*
584 * select CCR3
585 */
586 movb $CYRIX_CCR3, %al
587 outb $CYRIX_CRI
588
589 /*
590 * disable MAPEN and write CCR3
591 */
592 movb %cl, %al
593 outb $CYRIX_CRD
594
595 /*
596 * restore IMR
597 */
598 movb %bl, %al
599 outb $MIMR_PORT
600
601 /*
602 * test to see if cpuid available
603 */
604 pushfl
605 popl %ecx
606 movl %ecx, %eax
607 xorl $PS_ID, %eax / try complemented bit
608 pushl %eax
609 popfl
610 pushfl
611 popl %eax
612 cmpl %eax, %ecx
613 jne have_cpuid
614 jmp cpu_486
615
616 restore_PIC:
617 /*
618 * In case the motherboard/chipset vendor is ignoring line A1 of the
619 * I/O address, we set the PIC priorities to sane values.
620 */
621 movb $0xc7, %al / irq 7 lowest priority
622 outb $MCMD_PORT
623
624 restore_IMR:
625 movb %bl, %al
626 outb $MIMR_PORT
627 jmp cpu_486
628
629 have_cpuid:
630 /*
631 * cpuid instruction present
632 */
633 bts $X86FSET_CPUID, x86_featureset / Just to set; Ignore the CF
634 movl $0, %eax
635 cpuid
636
637 movl %ebx, cpu_vendor
638 movl %edx, cpu_vendor+4
639 movl %ecx, cpu_vendor+8
640
641 /*
642 * early cyrix cpus are somewhat strange and need to be
643 * probed in curious ways to determine their identity
644 */
645
646 leal cpu_vendor, %esi
647 leal CyrixInstead, %edi
648 movl $12, %ecx
649 repz
650 cmpsb
651 je vendor_is_cyrix
652
653 / let mlsetup()/cpuid_pass1() handle everything else in C
654
655 jmp cpu_done
656
657 is486:
658 /*
659 * test to see if a useful cpuid
660 */
661 testl %eax, %eax
662 jz isa486
663
664 movl $1, %eax
665 cpuid
666
667 movl %eax, %ebx
668 andl $0xF00, %ebx
669 cmpl $0x400, %ebx
670 je isa486
671
672 rep; ret /* use 2 byte return instruction */
673 /* AMD Software Optimization Guide - Section 6.2 */
674 isa486:
675 /*
676 * lose the return address
677 */
678 popl %eax
679 jmp cpu_486
680
681 vendor_is_cyrix:
682 call is486
683
684 /*
685 * Processor signature and feature flags for Cyrix are insane.
686 * BIOS can play with semi-documented registers, so cpuid must be used
687 * cautiously. Since we are Cyrix that has cpuid, we have DIR0 and DIR1
688 * Keep the family in %ebx and feature flags in %edx until not needed
689 */
690
691 /*
692 * read DIR0
693 */
694 movb $CYRIX_DIR0, %al
695 outb $CYRIX_CRI
696 inb $CYRIX_CRD
697
698 /*
699 * First we handle the cases where we are a 6x86 or 6x86L.
700 * The 6x86 is basically a 486, the only reliable bit in the
701 * feature flags is for FPU. The 6x86L is better, unfortunately
702 * there is no really good way to distinguish between these two
703 * cpu's. We are pessimistic and when in doubt assume 6x86.
704 */
705
706 cmpb $0x40, %al
707 jae maybeGX
708
709 /*
710 * We are an M1, either a 6x86 or 6x86L.
711 */
712 cmpb $0x30, %al
713 je maybe6x86L
714 cmpb $0x31, %al
715 je maybe6x86L
716 cmpb $0x34, %al
717 je maybe6x86L
718 cmpb $0x35, %al
719 je maybe6x86L
720
721 /*
722 * although it is possible that we are a 6x86L, the cpu and
723 * documentation are so buggy, we just do not care.
724 */
725 jmp likely6x86
726
727 maybe6x86L:
728 /*
729 * read DIR1
730 */
731 movb $CYRIX_DIR1, %al
732 outb $CYRIX_CRI
733 inb $CYRIX_CRD
734 cmpb $0x22, %al
735 jb likely6x86
736
737 /*
738 * We are a 6x86L, or at least a 6x86 with honest cpuid feature flags
739 */
740 movl $X86_TYPE_CYRIX_6x86L, x86_type
741 jmp coma_bug
742
743 likely6x86:
744 /*
745 * We are likely a 6x86, or a 6x86L without a way of knowing
746 *
747 * The 6x86 has NO Pentium or Pentium Pro compatible features even
748 * though it claims to be a Pentium Pro compatible!
749 *
750 * The 6x86 core used in the 6x86 may have most of the Pentium system
751 * registers and largely conform to the Pentium System Programming
752 * Reference. Documentation on these parts is long gone. Treat it as
753 * a crippled Pentium and hope for the best.
754 */
755
756 movl $X86_TYPE_CYRIX_6x86, x86_type
757 jmp coma_bug
758
759 maybeGX:
760 /*
761 * Now we check whether we are a MediaGX or GXm. We have particular
762 * reason for concern here. Even though most of the GXm's
763 * report having TSC in the cpuid feature flags, the TSC may be
764 * horribly broken. What is worse, is that MediaGX's are basically
765 * 486's while the good GXm's are more like Pentium Pro's!
766 */
767
768 cmpb $0x50, %al
769 jae maybeM2
770
771 /*
772 * We are either a MediaGX (sometimes called a Gx86) or GXm
773 */
774
775 cmpb $41, %al
776 je maybeMediaGX
777
778 cmpb $44, %al
779 jb maybeGXm
780
781 cmpb $47, %al
782 jbe maybeMediaGX
783
784 /*
785 * We do not honestly know what we are, so assume a MediaGX
786 */
787 jmp media_gx
788
789 maybeGXm:
790 /*
791 * It is still possible we are either a MediaGX or GXm, trust cpuid
792 * family should be 5 on a GXm
793 */
794 cmpl $0x500, %ebx
795 je GXm
796
797 /*
798 * BIOS/Cyrix might set family to 6 on a GXm
799 */
800 cmpl $0x600, %ebx
801 jne media_gx
802
803 GXm:
804 movl $X86_TYPE_CYRIX_GXm, x86_type
805 jmp cpu_done
806
807 maybeMediaGX:
808 /*
809 * read DIR1
810 */
811 movb $CYRIX_DIR1, %al
812 outb $CYRIX_CRI
813 inb $CYRIX_CRD
814
815 cmpb $0x30, %al
816 jae maybeGXm
817
818 /*
819 * we are a MediaGX for which we do not trust cpuid
820 */
821 media_gx:
822 movl $X86_TYPE_CYRIX_MediaGX, x86_type
823 jmp cpu_486
824
825 maybeM2:
826 /*
827 * Now we check whether we are a 6x86MX or MII. These cpu's are
828 * virtually identical, but we care because for the 6x86MX, we
829 * must work around the coma bug. Also for 6x86MX prior to revision
830 * 1.4, the TSC may have serious bugs.
831 */
832
833 cmpb $0x60, %al
834 jae maybeM3
835
836 /*
837 * family should be 6, but BIOS/Cyrix might set it to 5
838 */
839 cmpl $0x600, %ebx
840 ja cpu_486
841
842 /*
843 * read DIR1
844 */
845 movb $CYRIX_DIR1, %al
846 outb $CYRIX_CRI
847 inb $CYRIX_CRD
848
849 cmpb $0x8, %al
850 jb cyrix6x86MX
851 cmpb $0x80, %al
852 jb MII
853
854 cyrix6x86MX:
855 /*
856 * It is altogether unclear how the revision stamped on the cpu
857 * maps to the values in DIR0 and DIR1. Just assume TSC is broken.
858 */
859 movl $X86_TYPE_CYRIX_6x86MX, x86_type
860 jmp coma_bug
861
862 MII:
863 movl $X86_TYPE_CYRIX_MII, x86_type
864 likeMII:
865 jmp cpu_done
866
867 maybeM3:
868 /*
869 * We are some chip that we cannot identify yet, an MIII perhaps.
870 * We will be optimistic and hope that the chip is much like an MII,
871 * and that cpuid is sane. Cyrix seemed to have gotten it right in
872 * time for the MII, we can only hope it stayed that way.
873 * Maybe the BIOS or Cyrix is trying to hint at something
874 */
875 cmpl $0x500, %ebx
876 je GXm
877
878 cmpb $0x80, %al
879 jae likelyM3
880
881 /*
882 * Just test for the features Cyrix is known for
883 */
884
885 jmp MII
886
887 likelyM3:
888 /*
889 * DIR0 with values from 0x80 to 0x8f indicates a VIA Cyrix III, aka
890 * the Cyrix MIII. There may be parts later that use the same ranges
891 * for DIR0 with special values in DIR1, maybe the VIA CIII, but for
892 * now we will call anything with a DIR0 of 0x80 or higher an MIII.
893 * The MIII is supposed to support large pages, but we will believe
894 * it when we see it. For now we just enable and test for MII features.
895 */
896 movl $X86_TYPE_VIA_CYRIX_III, x86_type
897 jmp likeMII
898
899 coma_bug:
900
901 /*
902 * With NO_LOCK set to 0 in CCR1, the usual state that BIOS enforces, some
903 * bus cycles are issued with LOCK# asserted. With NO_LOCK set to 1, all bus
904 * cycles except page table accesses and interrupt ACK cycles do not assert
905 * LOCK#. xchgl is an instruction that asserts LOCK# if NO_LOCK is set to 0.
906 * Due to a bug in the cpu core involving over-optimization of branch
907 * prediction, register renaming, and execution of instructions down both the
908 * X and Y pipes for the xchgl instruction, short loops can be written that
909 * never de-assert LOCK# from one invocation of the loop to the next, ad
910 * infinitum. The undesirable effect of this situation is that interrupts are
911 * not serviced. The ideal workaround to this bug would be to set NO_LOCK to
912 * 1. Unfortunately bus cycles that would otherwise have asserted LOCK# no
913 * longer do, unless they are page table accesses or interrupt ACK cycles.
914 * With LOCK# not asserted, these bus cycles are now cached. This can cause
915 * undesirable behaviour if the ARR's are not configured correctly. Solaris
916 * does not configure the ARR's, nor does it provide any useful mechanism for
917 * doing so, thus the ideal workaround is not viable. Fortunately, the only
918 * known exploits for this bug involve the xchgl instruction specifically.
919 * There is a group of undocumented registers on Cyrix 6x86, 6x86L, and
920 * 6x86MX cpu's which can be used to specify one instruction as a serializing
921 * instruction. With the xchgl instruction serialized, LOCK# is still
922 * asserted, but it is the sole instruction for which LOCK# is asserted.
923 * There is now some added penalty for the xchgl instruction, but the usual
924 * bus locking is preserved. This ingenious workaround was discovered by
925 * disassembling a binary provided by Cyrix as a workaround for this bug on
926 * Windows, but its not documented anywhere by Cyrix, nor is the bug actually
927 * mentioned in any public errata! The only concern for this workaround is
928 * that there may be similar undiscovered bugs with other instructions that
929 * assert LOCK# that may be leveraged to similar ends. The fact that Cyrix
930 * fixed this bug sometime late in 1997 and no other exploits other than
931 * xchgl have been discovered is good indication that this workaround is
932 * reasonable.
933 */
934
935 .set CYRIX_DBR0, 0x30 / Debug Register 0
936 .set CYRIX_DBR1, 0x31 / Debug Register 1
937 .set CYRIX_DBR2, 0x32 / Debug Register 2
938 .set CYRIX_DBR3, 0x33 / Debug Register 3
939 .set CYRIX_DOR, 0x3c / Debug Opcode Register
940
941 /*
942 * What is known about DBR1, DBR2, DBR3, and DOR is that for normal
943 * cpu execution DBR1, DBR2, and DBR3 are set to 0. To obtain opcode
944 * serialization, DBR1, DBR2, and DBR3 are loaded with 0xb8, 0x7f,
945 * and 0xff. Then, DOR is loaded with the one byte opcode.
946 */
947
948 /*
949 * select CCR3
950 */
951 movb $CYRIX_CCR3, %al
952 outb $CYRIX_CRI
953
954 /*
955 * read CCR3 and mask out MAPEN
956 */
957 inb $CYRIX_CRD
958 andb $0xf, %al
959
960 /*
961 * save masked CCR3 in %ah
962 */
963 movb %al, %ah
964
965 /*
966 * select CCR3
967 */
968 movb $CYRIX_CCR3, %al
969 outb $CYRIX_CRI
970
971 /*
972 * enable MAPEN
973 */
974 movb %ah, %al
975 orb $0x10, %al
976 outb $CYRIX_CRD
977
978 /*
979 * read DBR0
980 */
981 movb $CYRIX_DBR0, %al
982 outb $CYRIX_CRI
983 inb $CYRIX_CRD
984
985 /*
986 * disable MATCH and save in %bh
987 */
988 orb $0x80, %al
989 movb %al, %bh
990
991 /*
992 * write DBR0
993 */
994 movb $CYRIX_DBR0, %al
995 outb $CYRIX_CRI
996 movb %bh, %al
997 outb $CYRIX_CRD
998
999 /*
1000 * write DBR1
1001 */
1002 movb $CYRIX_DBR1, %al
1003 outb $CYRIX_CRI
1004 movb $0xf8, %al
1005 outb $CYRIX_CRD
1006
1007 /*
1008 * write DBR2
1009 */
1010 movb $CYRIX_DBR2, %al
1011 outb $CYRIX_CRI
1012 movb $0x7f, %al
1013 outb $CYRIX_CRD
1014
1015 /*
1016 * write DBR3
1017 */
1018 movb $CYRIX_DBR3, %al
1019 outb $CYRIX_CRI
1020 xorb %al, %al
1021 outb $CYRIX_CRD
1022
1023 /*
1024 * write DOR
1025 */
1026 movb $CYRIX_DOR, %al
1027 outb $CYRIX_CRI
1028 movb $0x87, %al
1029 outb $CYRIX_CRD
1030
1031 /*
1032 * enable MATCH
1033 */
1034 movb $CYRIX_DBR0, %al
1035 outb $CYRIX_CRI
1036 movb %bh, %al
1037 andb $0x7f, %al
1038 outb $CYRIX_CRD
1039
1040 /*
1041 * disable MAPEN
1042 */
1043 movb $0xc3, %al
1044 outb $CYRIX_CRI
1045 movb %ah, %al
1046 outb $CYRIX_CRD
1047
1048 jmp cpu_done
1049
1050 cpu_done:
1051
1052 popfl /* Restore original FLAGS */
1053 popal /* Restore all registers */
1054
1055 #endif /* !__xpv */
1056
1057 /*
1058 * mlsetup(%esp) gets called.
1059 */
1060 pushl %esp
1061 call mlsetup
1062 addl $4, %esp
1063
1064 /*
1065 * We change our appearance to look like the real thread 0.
1066 * (NOTE: making ourselves to be a real thread may be a noop)
1067 * main() gets called. (NOTE: main() never returns).
1068 */
1069 call main
1070 /* NOTREACHED */
1071 pushl $__return_from_main
1072 call panic
1073
1074 /* NOTREACHED */
1075 cpu_486:
1076 pushl $__unsupported_cpu
1077 call panic
1078 SET_SIZE(_locore_start)
1079
1080 #endif /* __lint */
1081 #endif /* !__amd64 */
1082
1083
1084 /*
1085 * For stack layout, see privregs.h
1086 * When cmntrap gets called, the error code and trap number have been pushed.
1087 * When cmntrap_pushed gets called, the entire struct regs has been pushed.
1088 */
1089
1090 #if defined(__lint)
1091
1092 /* ARGSUSED */
1093 void
1094 cmntrap()
1095 {}
1096
1097 #else /* __lint */
1098
1099 .globl trap /* C handler called below */
1100
1101 #if defined(__amd64)
1102
1103 ENTRY_NP2(cmntrap, _cmntrap)
1104
1105 INTR_PUSH
1106
1107 ALTENTRY(cmntrap_pushed)
1108
1109 movq %rsp, %rbp
1110
1111 /*
1112 * - if this is a #pf i.e. T_PGFLT, %r15 is live
1113 * and contains the faulting address i.e. a copy of %cr2
1114 *
1115 * - if this is a #db i.e. T_SGLSTP, %r15 is live
1116 * and contains the value of %db6
1117 */
1118
1119 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1120 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1121 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1122
1189 movq %rdi, %r12
1190 call dtrace_instr_size
1191 addq %rax, %r12
1192 movq %r12, REGOFF_RIP(%rbp)
1193 INTR_POP
1194 call x86_md_clear
1195 jmp tr_iret_auto
1196 /*NOTREACHED*/
1197 3:
1198 leaq dtrace_badflags(%rip), %rdi
1199 xorl %eax, %eax
1200 call panic
1201 4:
1202 leaq dtrace_badtrap(%rip), %rdi
1203 xorl %eax, %eax
1204 call panic
1205 SET_SIZE(cmntrap_pushed)
1206 SET_SIZE(cmntrap)
1207 SET_SIZE(_cmntrap)
1208
1209 #elif defined(__i386)
1210
1211
1212 ENTRY_NP2(cmntrap, _cmntrap)
1213
1214 INTR_PUSH
1215
1216 ALTENTRY(cmntrap_pushed)
1217
1218 movl %esp, %ebp
1219
1220 /*
1221 * - if this is a #pf i.e. T_PGFLT, %esi is live
1222 * and contains the faulting address i.e. a copy of %cr2
1223 *
1224 * - if this is a #db i.e. T_SGLSTP, %esi is live
1225 * and contains the value of %db6
1226 */
1227
1228 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1229 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1230 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1231
1232 /*
1233 * We must first check if DTrace has set its NOFAULT bit. This
1234 * regrettably must happen before the trap stack is recorded, because
1235 * this requires a call to getpcstack() and may induce recursion if an
1236 * fbt::getpcstack: enabling is inducing the bad load.
1237 */
1238 movl %gs:CPU_ID, %eax
1239 shll $CPU_CORE_SHIFT, %eax
1240 addl $cpu_core, %eax
1241 movw CPUC_DTRACE_FLAGS(%eax), %cx
1242 testw $CPU_DTRACE_NOFAULT, %cx
1243 jnz .dtrace_induced
1244
1245 TRACE_STACK(%edi)
1246
1247 pushl %gs:CPU_ID
1248 pushl %esi /* fault address for PGFLTs */
1249 pushl %ebp /* ®s */
1250
1251 /*
1252 * We know that this isn't a DTrace non-faulting load; we can now safely
1253 * reenable interrupts. (In the case of pagefaults, we enter through an
1254 * interrupt gate.)
1255 */
1256 ENABLE_INTR_FLAGS
1257
1258 call trap /* trap(rp, addr, cpuid) handles all traps */
1259 addl $12, %esp /* get argument off stack */
1260 jmp _sys_rtt
1261
1262 .dtrace_induced:
1263 cmpw $KCS_SEL, REGOFF_CS(%ebp) /* test CS for user-mode trap */
1264 jne 3f /* if from user, panic */
1265
1266 cmpl $T_PGFLT, REGOFF_TRAPNO(%ebp)
1267 je 1f
1268
1269 cmpl $T_GPFLT, REGOFF_TRAPNO(%ebp)
1270 je 0f
1271
1272 cmpl $T_ZERODIV, REGOFF_TRAPNO(%ebp)
1273 jne 4f /* if not PF/GP/UD/DE, panic */
1274
1275 orw $CPU_DTRACE_DIVZERO, %cx
1276 movw %cx, CPUC_DTRACE_FLAGS(%eax)
1277 jmp 2f
1278
1279 0:
1280 /*
1281 * If we've taken a GPF, we don't (unfortunately) have the address that
1282 * induced the fault. So instead of setting the fault to BADADDR,
1283 * we'll set the fault to ILLOP.
1284 */
1285 orw $CPU_DTRACE_ILLOP, %cx
1286 movw %cx, CPUC_DTRACE_FLAGS(%eax)
1287 jmp 2f
1288 1:
1289 orw $CPU_DTRACE_BADADDR, %cx
1290 movw %cx, CPUC_DTRACE_FLAGS(%eax) /* set fault to bad addr */
1291 movl %esi, CPUC_DTRACE_ILLVAL(%eax)
1292 /* fault addr is illegal value */
1293 2:
1294 pushl REGOFF_EIP(%ebp)
1295 call dtrace_instr_size
1296 addl $4, %esp
1297 movl REGOFF_EIP(%ebp), %ecx
1298 addl %eax, %ecx
1299 movl %ecx, REGOFF_EIP(%ebp)
1300 INTR_POP_KERNEL
1301 IRET
1302 /*NOTREACHED*/
1303 3:
1304 pushl $dtrace_badflags
1305 call panic
1306 4:
1307 pushl $dtrace_badtrap
1308 call panic
1309 SET_SIZE(cmntrap)
1310 SET_SIZE(_cmntrap)
1311
1312 #endif /* __i386 */
1313
1314 /*
1315 * Declare a uintptr_t which has the size of _cmntrap to enable stack
1316 * traceback code to know when a regs structure is on the stack.
1317 */
1318 .globl _cmntrap_size
1319 .align CLONGSIZE
1320 _cmntrap_size:
1321 .NWORD . - _cmntrap
1322 .type _cmntrap_size, @object
1323
1324 dtrace_badflags:
1325 .string "bad DTrace flags"
1326
1327 dtrace_badtrap:
1328 .string "bad DTrace trap"
1329
1330 #endif /* __lint */
1331
1332 #if defined(__lint)
1333
1334 /* ARGSUSED */
1335 void
1336 cmninttrap()
1337 {}
1338
1339 #if !defined(__xpv)
1340 void
1341 bop_trap_handler(void)
1342 {}
1343 #endif
1344
1345 #else /* __lint */
1346
1347 .globl trap /* C handler called below */
1348
1349 #if defined(__amd64)
1350
1351 ENTRY_NP(cmninttrap)
1352
1353 INTR_PUSH
1354 INTGATE_INIT_KERNEL_FLAGS
1355
1356 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1357 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1358 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1359
1360 movq %rsp, %rbp
1361
1362 movl %gs:CPU_ID, %edx
1363 xorl %esi, %esi
1364 movq %rsp, %rdi
1365 call trap /* trap(rp, addr, cpuid) handles all traps */
1366 jmp _sys_rtt
1367 SET_SIZE(cmninttrap)
1368
1369 #if !defined(__xpv)
1370 /*
1371 * Handle traps early in boot. Just revectors into C quickly as
1372 * these are always fatal errors.
1373 *
1374 * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap().
1375 */
1376 ENTRY(bop_trap_handler)
1377 movq %rsp, %rdi
1378 sub $8, %rsp
1379 call bop_trap
1380 SET_SIZE(bop_trap_handler)
1381 #endif
1382
1383 #elif defined(__i386)
1384
1385 ENTRY_NP(cmninttrap)
1386
1387 INTR_PUSH
1388 INTGATE_INIT_KERNEL_FLAGS
1389
1390 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1391 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1392 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1393
1394 movl %esp, %ebp
1395
1396 TRACE_STACK(%edi)
1397
1398 pushl %gs:CPU_ID
1399 pushl $0
1400 pushl %ebp
1401 call trap /* trap(rp, addr, cpuid) handles all traps */
1402 addl $12, %esp
1403 jmp _sys_rtt
1404 SET_SIZE(cmninttrap)
1405
1406 #if !defined(__xpv)
1407 /*
1408 * Handle traps early in boot. Just revectors into C quickly as
1409 * these are always fatal errors.
1410 */
1411 ENTRY(bop_trap_handler)
1412 movl %esp, %eax
1413 pushl %eax
1414 call bop_trap
1415 SET_SIZE(bop_trap_handler)
1416 #endif
1417
1418 #endif /* __i386 */
1419
1420 #endif /* __lint */
1421
1422 #if defined(__lint)
1423
1424 /* ARGSUSED */
1425 void
1426 dtrace_trap()
1427 {}
1428
1429 #else /* __lint */
1430
1431 .globl dtrace_user_probe
1432
1433 #if defined(__amd64)
1434
1435 ENTRY_NP(dtrace_trap)
1436
1437 INTR_PUSH
1438
1439 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1440 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1441 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1442
1443 movq %rsp, %rbp
1444
1445 movl %gs:CPU_ID, %edx
1446 #if defined(__xpv)
1447 movq %gs:CPU_VCPU_INFO, %rsi
1448 movq VCPU_INFO_ARCH_CR2(%rsi), %rsi
1449 #else
1450 movq %cr2, %rsi
1451 #endif
1452 movq %rsp, %rdi
1453
1454 ENABLE_INTR_FLAGS
1455
1456 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
1457 jmp _sys_rtt
1458
1459 SET_SIZE(dtrace_trap)
1460
1461 #elif defined(__i386)
1462
1463 ENTRY_NP(dtrace_trap)
1464
1465 INTR_PUSH
1466
1467 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1468 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1469 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1470
1471 movl %esp, %ebp
1472
1473 pushl %gs:CPU_ID
1474 #if defined(__xpv)
1475 movl %gs:CPU_VCPU_INFO, %eax
1476 movl VCPU_INFO_ARCH_CR2(%eax), %eax
1477 #else
1478 movl %cr2, %eax
1479 #endif
1480 pushl %eax
1481 pushl %ebp
1482
1483 ENABLE_INTR_FLAGS
1484
1485 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
1486 addl $12, %esp /* get argument off stack */
1487
1488 jmp _sys_rtt
1489 SET_SIZE(dtrace_trap)
1490
1491 #endif /* __i386 */
1492
1493 #endif /* __lint */
1494
1495 /*
1496 * Return from _sys_trap routine.
1497 */
1498
1499 #if defined(__lint)
1500
1501 void
1502 lwp_rtt_initial(void)
1503 {}
1504
1505 void
1506 lwp_rtt(void)
1507 {}
1508
1509 void
1510 _sys_rtt(void)
1511 {}
1512
1513 #else /* __lint */
1514
1515 ENTRY_NP(lwp_rtt_initial)
1516 movq %gs:CPU_THREAD, %r15
1517 movq T_STACK(%r15), %rsp /* switch to the thread stack */
1518 movq %rsp, %rbp
1519 call __dtrace_probe___proc_start
1520 jmp _lwp_rtt
1521
1522 ENTRY_NP(lwp_rtt)
1523
1524 /*
1525 * r14 lwp
1526 * rdx lwp->lwp_procp
1527 * r15 curthread
1528 */
1529
1530 movq %gs:CPU_THREAD, %r15
1531 movq T_STACK(%r15), %rsp /* switch to the thread stack */
1532 movq %rsp, %rbp
1533 _lwp_rtt:
1534 call __dtrace_probe___proc_lwp__start
1619 * after _sys_rtt .
1620 */
1621 ALTENTRY(sr_sup)
1622 /*
1623 * Restore regs before doing iretq to kernel mode
1624 */
1625 INTR_POP
1626 jmp tr_iret_kernel
1627 .globl _sys_rtt_end
1628 _sys_rtt_end:
1629 /*NOTREACHED*/
1630 SET_SIZE(sr_sup)
1631 SET_SIZE(_sys_rtt_end)
1632 SET_SIZE(lwp_rtt)
1633 SET_SIZE(lwp_rtt_initial)
1634 SET_SIZE(_sys_rtt_ints_disabled)
1635 SET_SIZE(_sys_rtt)
1636 SET_SIZE(sys_rtt_syscall)
1637 SET_SIZE(sys_rtt_syscall32)
1638
1639 #endif /* __lint */
1640
1641 #if defined(__lint)
1642
1643 /*
1644 * So why do we have to deal with all this crud in the world of ia32?
1645 *
1646 * Basically there are four classes of ia32 implementations, those that do not
1647 * have a TSC, those that have a marginal TSC that is broken to the extent
1648 * that it is useless, those that have a marginal TSC that is not quite so
1649 * horribly broken and can be used with some care, and those that have a
1650 * reliable TSC. This crud has to be here in order to sift through all the
1651 * variants.
1652 */
1653
1654 /*ARGSUSED*/
1655 uint64_t
1656 freq_tsc(uint32_t *pit_counter)
1657 {
1658 return (0);
1659 }
1660
1661 #else /* __lint */
1662
1663 #if defined(__amd64)
1664
1665 /*
1666 * XX64 quick and dirty port from the i386 version. Since we
1667 * believe the amd64 tsc is more reliable, could this code be
1668 * simpler?
1669 */
1670 ENTRY_NP(freq_tsc)
1671 pushq %rbp
1672 movq %rsp, %rbp
1673 movq %rdi, %r9 /* save pit_counter */
1674 pushq %rbx
1675
1676 / We have a TSC, but we have no way in general to know how reliable it is.
1677 / Usually a marginal TSC behaves appropriately unless not enough time
1678 / elapses between reads. A reliable TSC can be read as often and as rapidly
1679 / as desired. The simplistic approach of reading the TSC counter and
1680 / correlating to the PIT counter cannot be naively followed. Instead estimates
1681 / have to be taken to successively refine a guess at the speed of the cpu
1682 / and then the TSC and PIT counter are correlated. In practice very rarely
1683 / is more than one quick loop required for an estimate. Measures have to be
1684 / taken to prevent the PIT counter from wrapping beyond its resolution and for
1850 movl %ecx, %eax
1851 subl %esi, %eax
1852 sbbl %edi, %edx
1853
1854 jmp freq_tsc_end
1855
1856 freq_tsc_too_fast:
1857 / return 0 as a 64 bit quantity
1858 xorl %eax, %eax
1859 xorl %edx, %edx
1860
1861 freq_tsc_end:
1862 shlq $32, %rdx
1863 orq %rdx, %rax
1864
1865 popq %rbx
1866 leaveq
1867 ret
1868 SET_SIZE(freq_tsc)
1869
1870 #elif defined(__i386)
1871
1872 ENTRY_NP(freq_tsc)
1873 pushl %ebp
1874 movl %esp, %ebp
1875 pushl %edi
1876 pushl %esi
1877 pushl %ebx
1878
1879 / We have a TSC, but we have no way in general to know how reliable it is.
1880 / Usually a marginal TSC behaves appropriately unless not enough time
1881 / elapses between reads. A reliable TSC can be read as often and as rapidly
1882 / as desired. The simplistic approach of reading the TSC counter and
1883 / correlating to the PIT counter cannot be naively followed. Instead estimates
1884 / have to be taken to successively refine a guess at the speed of the cpu
1885 / and then the TSC and PIT counter are correlated. In practice very rarely
1886 / is more than one quick loop required for an estimate. Measures have to be
1887 / taken to prevent the PIT counter from wrapping beyond its resolution and for
1888 / measuring the clock rate of very fast processors.
1889 /
1890 / The following constant can be tuned. It should be such that the loop does
1891 / not take too many nor too few PIT counts to execute. If this value is too
1892 / large, then on slow machines the loop will take a long time, or the PIT
1893 / counter may even wrap. If this value is too small, then on fast machines
1894 / the PIT counter may count so few ticks that the resolution of the PIT
1895 / itself causes a bad guess. Because this code is used in machines with
1896 / marginal TSC's and/or IO, if this value is too small on those, it may
1897 / cause the calculated cpu frequency to vary slightly from boot to boot.
1898 /
1899 / In all cases even if this constant is set inappropriately, the algorithm
1900 / will still work and the caller should be able to handle variances in the
1901 / calculation of cpu frequency, but the calculation will be inefficient and
1902 / take a disproportionate amount of time relative to a well selected value.
1903 / As the slowest supported cpu becomes faster, this constant should be
1904 / carefully increased.
1905
1906 movl $0x8000, %ecx
1907
1908 / to make sure the instruction cache has been warmed
1909 clc
1910
1911 jmp freq_tsc_loop
1912
1913 / The following block of code up to and including the latching of the PIT
1914 / counter after freq_tsc_perf_loop is very critical and very carefully
1915 / written, it should only be modified with great care. freq_tsc_loop to
1916 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
1917 / freq_tsc_perf_loop up to the unlatching of the PIT counter.
1918
1919 .align 32
1920 freq_tsc_loop:
1921 / save the loop count in %ebx
1922 movl %ecx, %ebx
1923
1924 / initialize the PIT counter and start a count down
1925 movb $PIT_LOADMODE, %al
1926 outb $PITCTL_PORT
1927 movb $0xff, %al
1928 outb $PITCTR0_PORT
1929 outb $PITCTR0_PORT
1930
1931 / read the TSC and store the TS in %edi:%esi
1932 rdtsc
1933 movl %eax, %esi
1934
1935 freq_tsc_perf_loop:
1936 movl %edx, %edi
1937 movl %eax, %esi
1938 movl %edx, %edi
1939 loop freq_tsc_perf_loop
1940
1941 / read the TSC and store the LSW in %ecx
1942 rdtsc
1943 movl %eax, %ecx
1944
1945 / latch the PIT counter and status
1946 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
1947 outb $PITCTL_PORT
1948
1949 / remember if the icache has been warmed
1950 setc %ah
1951
1952 / read the PIT status
1953 inb $PITCTR0_PORT
1954 shll $8, %eax
1955
1956 / read PIT count
1957 inb $PITCTR0_PORT
1958 shll $8, %eax
1959 inb $PITCTR0_PORT
1960 bswap %eax
1961
1962 / check to see if the PIT count was loaded into the CE
1963 btw $_CONST(PITSTAT_NULLCNT+8), %ax
1964 jc freq_tsc_increase_count
1965
1966 / check to see if PIT counter wrapped
1967 btw $_CONST(PITSTAT_OUTPUT+8), %ax
1968 jnc freq_tsc_pit_did_not_wrap
1969
1970 / halve count
1971 shrl $1, %ebx
1972 movl %ebx, %ecx
1973
1974 / the instruction cache has been warmed
1975 stc
1976
1977 jmp freq_tsc_loop
1978
1979 freq_tsc_increase_count:
1980 shll $1, %ebx
1981 jc freq_tsc_too_fast
1982
1983 movl %ebx, %ecx
1984
1985 / the instruction cache has been warmed
1986 stc
1987
1988 jmp freq_tsc_loop
1989
1990 freq_tsc_pit_did_not_wrap:
1991 roll $16, %eax
1992
1993 cmpw $0x2000, %ax
1994 notw %ax
1995 jb freq_tsc_sufficient_duration
1996
1997 freq_tsc_calculate:
1998 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
1999 / then on the second CLK pulse the CE is decremented, therefore mode 0
2000 / is really a (count + 1) counter, ugh
2001 xorl %esi, %esi
2002 movw %ax, %si
2003 incl %esi
2004
2005 movl $0xf000, %eax
2006 mull %ebx
2007
2008 / tuck away (target_pit_count * loop_count)
2009 movl %edx, %ecx
2010 movl %eax, %ebx
2011
2012 movl %esi, %eax
2013 movl $0xffffffff, %edx
2014 mull %edx
2015
2016 addl %esi, %eax
2017 adcl $0, %edx
2018
2019 cmpl %ecx, %edx
2020 ja freq_tsc_div_safe
2021 jb freq_tsc_too_fast
2022
2023 cmpl %ebx, %eax
2024 jbe freq_tsc_too_fast
2025
2026 freq_tsc_div_safe:
2027 movl %ecx, %edx
2028 movl %ebx, %eax
2029
2030 movl %esi, %ecx
2031 divl %ecx
2032
2033 movl %eax, %ecx
2034
2035 / the instruction cache has been warmed
2036 stc
2037
2038 jmp freq_tsc_loop
2039
2040 freq_tsc_sufficient_duration:
2041 / test to see if the icache has been warmed
2042 btl $16, %eax
2043 jnc freq_tsc_calculate
2044
2045 / recall mode 0 is a (count + 1) counter
2046 andl $0xffff, %eax
2047 incl %eax
2048
2049 / save the number of PIT counts
2050 movl 8(%ebp), %ebx
2051 movl %eax, (%ebx)
2052
2053 / calculate the number of TS's that elapsed
2054 movl %ecx, %eax
2055 subl %esi, %eax
2056 sbbl %edi, %edx
2057
2058 jmp freq_tsc_end
2059
2060 freq_tsc_too_fast:
2061 / return 0 as a 64 bit quantity
2062 xorl %eax, %eax
2063 xorl %edx, %edx
2064
2065 freq_tsc_end:
2066 popl %ebx
2067 popl %esi
2068 popl %edi
2069 popl %ebp
2070 ret
2071 SET_SIZE(freq_tsc)
2072
2073 #endif /* __i386 */
2074 #endif /* __lint */
2075
2076 #if !defined(__amd64)
2077 #if defined(__lint)
2078
2079 /*
2080 * We do not have a TSC so we use a block of instructions with well known
2081 * timings.
2082 */
2083
2084 /*ARGSUSED*/
2085 uint64_t
2086 freq_notsc(uint32_t *pit_counter)
2087 {
2088 return (0);
2089 }
2090
2091 #else /* __lint */
2092 ENTRY_NP(freq_notsc)
2093 pushl %ebp
2094 movl %esp, %ebp
2095 pushl %edi
2096 pushl %esi
2097 pushl %ebx
2098
2099 / initial count for the idivl loop
2100 movl $0x1000, %ecx
2101
2102 / load the divisor
2103 movl $1, %ebx
2104
2105 jmp freq_notsc_loop
2106
2107 .align 16
2108 freq_notsc_loop:
2109 / set high 32 bits of dividend to zero
2110 xorl %edx, %edx
2111
2112 / save the loop count in %edi
2113 movl %ecx, %edi
2114
2115 / initialize the PIT counter and start a count down
2116 movb $PIT_LOADMODE, %al
2117 outb $PITCTL_PORT
2118 movb $0xff, %al
2119 outb $PITCTR0_PORT
2120 outb $PITCTR0_PORT
2121
2122 / set low 32 bits of dividend to zero
2123 xorl %eax, %eax
2124
2125 / It is vital that the arguments to idivl be set appropriately because on some
2126 / cpu's this instruction takes more or less clock ticks depending on its
2127 / arguments.
2128 freq_notsc_perf_loop:
2129 idivl %ebx
2130 idivl %ebx
2131 idivl %ebx
2132 idivl %ebx
2133 idivl %ebx
2134 loop freq_notsc_perf_loop
2135
2136 / latch the PIT counter and status
2137 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
2138 outb $PITCTL_PORT
2139
2140 / read the PIT status
2141 inb $PITCTR0_PORT
2142 shll $8, %eax
2143
2144 / read PIT count
2145 inb $PITCTR0_PORT
2146 shll $8, %eax
2147 inb $PITCTR0_PORT
2148 bswap %eax
2149
2150 / check to see if the PIT count was loaded into the CE
2151 btw $_CONST(PITSTAT_NULLCNT+8), %ax
2152 jc freq_notsc_increase_count
2153
2154 / check to see if PIT counter wrapped
2155 btw $_CONST(PITSTAT_OUTPUT+8), %ax
2156 jnc freq_notsc_pit_did_not_wrap
2157
2158 / halve count
2159 shrl $1, %edi
2160 movl %edi, %ecx
2161
2162 jmp freq_notsc_loop
2163
2164 freq_notsc_increase_count:
2165 shll $1, %edi
2166 jc freq_notsc_too_fast
2167
2168 movl %edi, %ecx
2169
2170 jmp freq_notsc_loop
2171
2172 freq_notsc_pit_did_not_wrap:
2173 shrl $16, %eax
2174
2175 cmpw $0x2000, %ax
2176 notw %ax
2177 jb freq_notsc_sufficient_duration
2178
2179 freq_notsc_calculate:
2180 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
2181 / then on the second CLK pulse the CE is decremented, therefore mode 0
2182 / is really a (count + 1) counter, ugh
2183 xorl %esi, %esi
2184 movw %ax, %si
2185 incl %esi
2186
2187 movl %edi, %eax
2188 movl $0xf000, %ecx
2189 mull %ecx
2190
2191 / tuck away (target_pit_count * loop_count)
2192 movl %edx, %edi
2193 movl %eax, %ecx
2194
2195 movl %esi, %eax
2196 movl $0xffffffff, %edx
2197 mull %edx
2198
2199 addl %esi, %eax
2200 adcl $0, %edx
2201
2202 cmpl %edi, %edx
2203 ja freq_notsc_div_safe
2204 jb freq_notsc_too_fast
2205
2206 cmpl %ecx, %eax
2207 jbe freq_notsc_too_fast
2208
2209 freq_notsc_div_safe:
2210 movl %edi, %edx
2211 movl %ecx, %eax
2212
2213 movl %esi, %ecx
2214 divl %ecx
2215
2216 movl %eax, %ecx
2217
2218 jmp freq_notsc_loop
2219
2220 freq_notsc_sufficient_duration:
2221 / recall mode 0 is a (count + 1) counter
2222 incl %eax
2223
2224 / save the number of PIT counts
2225 movl 8(%ebp), %ebx
2226 movl %eax, (%ebx)
2227
2228 / calculate the number of cpu clock ticks that elapsed
2229 cmpl $X86_VENDOR_Cyrix, x86_vendor
2230 jz freq_notsc_notcyrix
2231
2232 / freq_notsc_perf_loop takes 86 clock cycles on Cyrix 6x86 cores
2233 movl $86, %eax
2234 jmp freq_notsc_calculate_tsc
2235
2236 freq_notsc_notcyrix:
2237 / freq_notsc_perf_loop takes 237 clock cycles on Intel Pentiums
2238 movl $237, %eax
2239
2240 freq_notsc_calculate_tsc:
2241 mull %edi
2242
2243 jmp freq_notsc_end
2244
2245 freq_notsc_too_fast:
2246 / return 0 as a 64 bit quantity
2247 xorl %eax, %eax
2248 xorl %edx, %edx
2249
2250 freq_notsc_end:
2251 popl %ebx
2252 popl %esi
2253 popl %edi
2254 popl %ebp
2255
2256 ret
2257 SET_SIZE(freq_notsc)
2258
2259 #endif /* __lint */
2260 #endif /* !__amd64 */
2261
2262 #if !defined(__lint)
2263 .data
2264 #if !defined(__amd64)
2265 .align 4
2266 cpu_vendor:
2267 .long 0, 0, 0 /* Vendor ID string returned */
2268
2269 .globl CyrixInstead
2270
2271 .globl x86_featureset
2272 .globl x86_type
2273 .globl x86_vendor
2274 #endif
2275
2276 #endif /* __lint */
|
26 * Copyright 2019 Joyent, Inc.
27 */
28
29 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
31 /* All Rights Reserved */
32
33 /* Copyright (c) 1987, 1988 Microsoft Corporation */
34 /* All Rights Reserved */
35
36
37 #include <sys/asm_linkage.h>
38 #include <sys/asm_misc.h>
39 #include <sys/regset.h>
40 #include <sys/privregs.h>
41 #include <sys/psw.h>
42 #include <sys/reboot.h>
43 #include <sys/x86_archext.h>
44 #include <sys/machparam.h>
45
46 #include <sys/segments.h>
47 #include <sys/pcb.h>
48 #include <sys/trap.h>
49 #include <sys/ftrace.h>
50 #include <sys/traptrace.h>
51 #include <sys/clock.h>
52 #include <sys/cmn_err.h>
53 #include <sys/pit.h>
54 #include <sys/panic.h>
55
56 #if defined(__xpv)
57 #include <sys/hypervisor.h>
58 #endif
59
60 #include "assym.h"
61
62 /*
63 * Our assumptions:
64 * - We are running in protected-paged mode.
65 * - Interrupts are disabled.
103 .globl panic
104 .globl t0stack
105 .globl t0
106 .globl sysp
107 .globl edata
108
109 /*
110 * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
111 */
112 .globl bootops
113 .globl bootopsp
114
115 /*
116 * NOTE: t0stack should be the first thing in the data section so that
117 * if it ever overflows, it will fault on the last kernel text page.
118 */
119 .data
120 .comm t0stack, DEFAULTSTKSZ, 32
121 .comm t0, 4094, 32
122
123
124 /*
125 * kobj_init() vectors us back to here with (note) a slightly different
126 * set of arguments than _start is given (see lint prototypes above).
127 *
128 * XXX Make this less vile, please.
129 */
130 ENTRY_NP(_locore_start)
131
132 /*
133 * %rdi = boot services (should die someday)
134 * %rdx = bootops
135 * end
136 */
137
138 leaq edata(%rip), %rbp /* reference edata for ksyms */
139 movq $0, (%rbp) /* limit stack back trace */
140
141 /*
142 * Initialize our stack pointer to the thread 0 stack (t0stack)
143 * and leave room for a "struct regs" for lwp0. Note that the
186 */
187 bts $X86FSET_CPUID, x86_featureset(%rip)
188
189 /*
190 * mlsetup() gets called with a struct regs as argument, while
191 * main takes no args and should never return.
192 */
193 xorl %ebp, %ebp
194 movq %rsp, %rdi
195 pushq %rbp
196 /* (stack pointer now aligned on 16-byte boundary right here) */
197 movq %rsp, %rbp
198 call mlsetup
199 call main
200 /* NOTREACHED */
201 leaq __return_from_main(%rip), %rdi
202 xorl %eax, %eax
203 call panic
204 SET_SIZE(_locore_start)
205
206 __return_from_main:
207 .string "main() returned"
208 __unsupported_cpu:
209 .string "486 style cpu detected - no longer supported!"
210
211 #if defined(DEBUG)
212 _no_pending_updates:
213 .string "locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
214 #endif
215
216 /*
217 * For stack layout, see privregs.h
218 * When cmntrap gets called, the error code and trap number have been pushed.
219 * When cmntrap_pushed gets called, the entire struct regs has been pushed.
220 */
221
222 .globl trap /* C handler called below */
223
224 ENTRY_NP2(cmntrap, _cmntrap)
225
226 INTR_PUSH
227
228 ALTENTRY(cmntrap_pushed)
229
230 movq %rsp, %rbp
231
232 /*
233 * - if this is a #pf i.e. T_PGFLT, %r15 is live
234 * and contains the faulting address i.e. a copy of %cr2
235 *
236 * - if this is a #db i.e. T_SGLSTP, %r15 is live
237 * and contains the value of %db6
238 */
239
240 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
241 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
242 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
243
310 movq %rdi, %r12
311 call dtrace_instr_size
312 addq %rax, %r12
313 movq %r12, REGOFF_RIP(%rbp)
314 INTR_POP
315 call x86_md_clear
316 jmp tr_iret_auto
317 /*NOTREACHED*/
318 3:
319 leaq dtrace_badflags(%rip), %rdi
320 xorl %eax, %eax
321 call panic
322 4:
323 leaq dtrace_badtrap(%rip), %rdi
324 xorl %eax, %eax
325 call panic
326 SET_SIZE(cmntrap_pushed)
327 SET_SIZE(cmntrap)
328 SET_SIZE(_cmntrap)
329
330 /*
331 * Declare a uintptr_t which has the size of _cmntrap to enable stack
332 * traceback code to know when a regs structure is on the stack.
333 */
334 .globl _cmntrap_size
335 .align CLONGSIZE
336 _cmntrap_size:
337 .NWORD . - _cmntrap
338 .type _cmntrap_size, @object
339
340 dtrace_badflags:
341 .string "bad DTrace flags"
342
343 dtrace_badtrap:
344 .string "bad DTrace trap"
345
346 .globl trap /* C handler called below */
347
348 ENTRY_NP(cmninttrap)
349
350 INTR_PUSH
351 INTGATE_INIT_KERNEL_FLAGS
352
353 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
354 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
355 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
356
357 movq %rsp, %rbp
358
359 movl %gs:CPU_ID, %edx
360 xorl %esi, %esi
361 movq %rsp, %rdi
362 call trap /* trap(rp, addr, cpuid) handles all traps */
363 jmp _sys_rtt
364 SET_SIZE(cmninttrap)
365
366 #if !defined(__xpv)
367 /*
368 * Handle traps early in boot. Just revectors into C quickly as
369 * these are always fatal errors.
370 *
371 * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap().
372 */
373 ENTRY(bop_trap_handler)
374 movq %rsp, %rdi
375 sub $8, %rsp
376 call bop_trap
377 SET_SIZE(bop_trap_handler)
378 #endif
379
380 .globl dtrace_user_probe
381
382 ENTRY_NP(dtrace_trap)
383
384 INTR_PUSH
385
386 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
387 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
388 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
389
390 movq %rsp, %rbp
391
392 movl %gs:CPU_ID, %edx
393 #if defined(__xpv)
394 movq %gs:CPU_VCPU_INFO, %rsi
395 movq VCPU_INFO_ARCH_CR2(%rsi), %rsi
396 #else
397 movq %cr2, %rsi
398 #endif
399 movq %rsp, %rdi
400
401 ENABLE_INTR_FLAGS
402
403 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
404 jmp _sys_rtt
405
406 SET_SIZE(dtrace_trap)
407
408 /*
409 * Return from _sys_trap routine.
410 */
411
412 ENTRY_NP(lwp_rtt_initial)
413 movq %gs:CPU_THREAD, %r15
414 movq T_STACK(%r15), %rsp /* switch to the thread stack */
415 movq %rsp, %rbp
416 call __dtrace_probe___proc_start
417 jmp _lwp_rtt
418
419 ENTRY_NP(lwp_rtt)
420
421 /*
422 * r14 lwp
423 * rdx lwp->lwp_procp
424 * r15 curthread
425 */
426
427 movq %gs:CPU_THREAD, %r15
428 movq T_STACK(%r15), %rsp /* switch to the thread stack */
429 movq %rsp, %rbp
430 _lwp_rtt:
431 call __dtrace_probe___proc_lwp__start
516 * after _sys_rtt .
517 */
518 ALTENTRY(sr_sup)
519 /*
520 * Restore regs before doing iretq to kernel mode
521 */
522 INTR_POP
523 jmp tr_iret_kernel
524 .globl _sys_rtt_end
525 _sys_rtt_end:
526 /*NOTREACHED*/
527 SET_SIZE(sr_sup)
528 SET_SIZE(_sys_rtt_end)
529 SET_SIZE(lwp_rtt)
530 SET_SIZE(lwp_rtt_initial)
531 SET_SIZE(_sys_rtt_ints_disabled)
532 SET_SIZE(_sys_rtt)
533 SET_SIZE(sys_rtt_syscall)
534 SET_SIZE(sys_rtt_syscall32)
535
536 /*
537 * XX64 quick and dirty port from the i386 version. Since we
538 * believe the amd64 tsc is more reliable, could this code be
539 * simpler?
540 */
541 ENTRY_NP(freq_tsc)
542 pushq %rbp
543 movq %rsp, %rbp
544 movq %rdi, %r9 /* save pit_counter */
545 pushq %rbx
546
547 / We have a TSC, but we have no way in general to know how reliable it is.
548 / Usually a marginal TSC behaves appropriately unless not enough time
549 / elapses between reads. A reliable TSC can be read as often and as rapidly
550 / as desired. The simplistic approach of reading the TSC counter and
551 / correlating to the PIT counter cannot be naively followed. Instead estimates
552 / have to be taken to successively refine a guess at the speed of the cpu
553 / and then the TSC and PIT counter are correlated. In practice very rarely
554 / is more than one quick loop required for an estimate. Measures have to be
555 / taken to prevent the PIT counter from wrapping beyond its resolution and for
721 movl %ecx, %eax
722 subl %esi, %eax
723 sbbl %edi, %edx
724
725 jmp freq_tsc_end
726
727 freq_tsc_too_fast:
728 / return 0 as a 64 bit quantity
729 xorl %eax, %eax
730 xorl %edx, %edx
731
732 freq_tsc_end:
733 shlq $32, %rdx
734 orq %rdx, %rax
735
736 popq %rbx
737 leaveq
738 ret
739 SET_SIZE(freq_tsc)
740
|