1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright 2019 Joyent, Inc.
  27  */
  28 
  29 /*      Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
  30 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T       */
  31 /*        All Rights Reserved                                   */
  32 
  33 /*      Copyright (c) 1987, 1988 Microsoft Corporation          */
  34 /*        All Rights Reserved                                   */
  35 
  36 
  37 #include <sys/asm_linkage.h>
  38 #include <sys/asm_misc.h>
  39 #include <sys/regset.h>
  40 #include <sys/privregs.h>
  41 #include <sys/psw.h>
  42 #include <sys/reboot.h>
  43 #include <sys/x86_archext.h>
  44 #include <sys/machparam.h>
  45 
  46 #include <sys/segments.h>
  47 #include <sys/pcb.h>
  48 #include <sys/trap.h>
  49 #include <sys/ftrace.h>
  50 #include <sys/traptrace.h>
  51 #include <sys/clock.h>
  52 #include <sys/cmn_err.h>
  53 #include <sys/pit.h>
  54 #include <sys/panic.h>
  55 
  56 #if defined(__xpv)
  57 #include <sys/hypervisor.h>
  58 #endif
  59 
  60 #include "assym.h"
  61 
  62 /*
  63  * Our assumptions:
  64  *      - We are running in protected-paged mode.
  65  *      - Interrupts are disabled.
  66  *      - The GDT and IDT are the callers; we need our copies.
  67  *      - The kernel's text, initialized data and bss are mapped.
  68  *
  69  * Our actions:
  70  *      - Save arguments
  71  *      - Initialize our stack pointer to the thread 0 stack (t0stack)
  72  *        and leave room for a phony "struct regs".
  73  *      - Our GDT and IDT need to get munged.
  74  *      - Since we are using the boot's GDT descriptors, we need
  75  *        to copy them into our GDT before we switch to ours.
  76  *      - We start using our GDT by loading correct values in the
  77  *        selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
  78  *        gs=KGS_SEL).
  79  *      - The default LDT entry for syscall is set.
  80  *      - We load the default LDT into the hardware LDT register.
  81  *      - We load the default TSS into the hardware task register.
  82  *      - Check for cpu type, i.e. 486 vs. P5 vs. P6 etc.
  83  *      - mlsetup(%esp) gets called.
  84  *      - We change our appearance to look like the real thread 0.
  85  *        (NOTE: making ourselves to be a real thread may be a noop)
  86  *      - main() gets called.  (NOTE: main() never returns).
  87  *
  88  * NOW, the real code!
  89  */
  90         /*
  91          * The very first thing in the kernel's text segment must be a jump
  92          * to the os/fakebop.c startup code.
  93          */
  94         .text
  95         jmp     _start
  96 
  97         /*
  98          * Globals:
  99          */
 100         .globl  _locore_start
 101         .globl  mlsetup
 102         .globl  main
 103         .globl  panic
 104         .globl  t0stack
 105         .globl  t0
 106         .globl  sysp
 107         .globl  edata
 108 
 109         /*
 110          * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
 111          */
 112         .globl  bootops
 113         .globl  bootopsp
 114 
 115         /*
 116          * NOTE: t0stack should be the first thing in the data section so that
 117          * if it ever overflows, it will fault on the last kernel text page.
 118          */
 119         .data
 120         .comm   t0stack, DEFAULTSTKSZ, 32
 121         .comm   t0, 4094, 32
 122 
 123 
 124         /*
 125          * kobj_init() vectors us back to here with (note) a slightly different
 126          * set of arguments than _start is given (see lint prototypes above).
 127          *
 128          * XXX  Make this less vile, please.
 129          */
 130         ENTRY_NP(_locore_start)
 131 
 132         /*
 133          * %rdi = boot services (should die someday)
 134          * %rdx = bootops
 135          * end
 136          */
 137 
 138         leaq    edata(%rip), %rbp       /* reference edata for ksyms */
 139         movq    $0, (%rbp)              /* limit stack back trace */
 140 
 141         /*
 142          * Initialize our stack pointer to the thread 0 stack (t0stack)
 143          * and leave room for a "struct regs" for lwp0.  Note that the
 144          * stack doesn't actually align to a 16-byte boundary until just
 145          * before we call mlsetup because we want to use %rsp to point at
 146          * our regs structure.
 147          */
 148         leaq    t0stack(%rip), %rsp
 149         addq    $_CONST(DEFAULTSTKSZ - REGSIZE), %rsp
 150 #if (REGSIZE & 15) == 0
 151         subq    $8, %rsp
 152 #endif
 153         /*
 154          * Save call back for special x86 boot services vector
 155          */
 156         movq    %rdi, sysp(%rip)
 157 
 158         movq    %rdx, bootops(%rip)             /* save bootops */
 159         movq    $bootops, bootopsp(%rip)
 160 
 161         /*
 162          * Save arguments and flags, if only for debugging ..
 163          */
 164         movq    %rdi, REGOFF_RDI(%rsp)
 165         movq    %rsi, REGOFF_RSI(%rsp)
 166         movq    %rdx, REGOFF_RDX(%rsp)
 167         movq    %rcx, REGOFF_RCX(%rsp)
 168         movq    %r8, REGOFF_R8(%rsp)
 169         movq    %r9, REGOFF_R9(%rsp)
 170         pushf
 171         popq    %r11
 172         movq    %r11, REGOFF_RFL(%rsp)
 173 
 174 #if !defined(__xpv)
 175         /*
 176          * Enable write protect and alignment check faults.
 177          */
 178         movq    %cr0, %rax
 179         orq     $_CONST(CR0_WP|CR0_AM), %rax
 180         andq    $_BITNOT(CR0_WT|CR0_CE), %rax
 181         movq    %rax, %cr0
 182 #endif  /* __xpv */
 183 
 184         /*
 185          * (We just assert this works by virtue of being here)
 186          */
 187         bts     $X86FSET_CPUID, x86_featureset(%rip)
 188 
 189         /*
 190          * mlsetup() gets called with a struct regs as argument, while
 191          * main takes no args and should never return.
 192          */
 193         xorl    %ebp, %ebp
 194         movq    %rsp, %rdi
 195         pushq   %rbp
 196         /* (stack pointer now aligned on 16-byte boundary right here) */
 197         movq    %rsp, %rbp
 198         call    mlsetup
 199         call    main
 200         /* NOTREACHED */
 201         leaq    __return_from_main(%rip), %rdi
 202         xorl    %eax, %eax
 203         call    panic
 204         SET_SIZE(_locore_start)
 205 
 206 __return_from_main:
 207         .string "main() returned"
 208 __unsupported_cpu:
 209         .string "486 style cpu detected - no longer supported!"
 210 
 211 #if defined(DEBUG)
 212 _no_pending_updates:
 213         .string "locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
 214 #endif
 215 
 216 /*
 217  *  For stack layout, see privregs.h
 218  *  When cmntrap gets called, the error code and trap number have been pushed.
 219  *  When cmntrap_pushed gets called, the entire struct regs has been pushed.
 220  */
 221 
 222         .globl  trap            /* C handler called below */
 223 
 224         ENTRY_NP2(cmntrap, _cmntrap)
 225 
 226         INTR_PUSH
 227 
 228         ALTENTRY(cmntrap_pushed)
 229 
 230         movq    %rsp, %rbp
 231 
 232         /*
 233          * - if this is a #pf i.e. T_PGFLT, %r15 is live
 234          *   and contains the faulting address i.e. a copy of %cr2
 235          *
 236          * - if this is a #db i.e. T_SGLSTP, %r15 is live
 237          *   and contains the value of %db6
 238          */
 239 
 240         TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
 241         TRACE_REGS(%rdi, %rsp, %rbx, %rcx)      /* Uses label 9 */
 242         TRACE_STAMP(%rdi)               /* Clobbers %eax, %edx, uses 9 */
 243 
 244         /*
 245          * We must first check if DTrace has set its NOFAULT bit.  This
 246          * regrettably must happen before the trap stack is recorded, because
 247          * this requires a call to getpcstack() and may induce recursion if an
 248          * fbt::getpcstack: enabling is inducing the bad load.
 249          */
 250         movl    %gs:CPU_ID, %eax
 251         shlq    $CPU_CORE_SHIFT, %rax
 252         leaq    cpu_core(%rip), %r8
 253         addq    %r8, %rax
 254         movw    CPUC_DTRACE_FLAGS(%rax), %cx
 255         testw   $CPU_DTRACE_NOFAULT, %cx
 256         jnz     .dtrace_induced
 257 
 258         TRACE_STACK(%rdi)
 259 
 260         movq    %rbp, %rdi
 261         movq    %r15, %rsi
 262         movl    %gs:CPU_ID, %edx
 263 
 264         /*
 265          * We know that this isn't a DTrace non-faulting load; we can now safely
 266          * reenable interrupts.  (In the case of pagefaults, we enter through an
 267          * interrupt gate.)
 268          */
 269         ENABLE_INTR_FLAGS
 270 
 271         call    trap            /* trap(rp, addr, cpuid) handles all traps */
 272         jmp     _sys_rtt
 273 
 274 .dtrace_induced:
 275         cmpw    $KCS_SEL, REGOFF_CS(%rbp)       /* test CS for user-mode trap */
 276         jne     3f                              /* if from user, panic */
 277 
 278         cmpl    $T_PGFLT, REGOFF_TRAPNO(%rbp)
 279         je      1f
 280 
 281         cmpl    $T_GPFLT, REGOFF_TRAPNO(%rbp)
 282         je      0f
 283 
 284         cmpl    $T_ILLINST, REGOFF_TRAPNO(%rbp)
 285         je      0f
 286 
 287         cmpl    $T_ZERODIV, REGOFF_TRAPNO(%rbp)
 288         jne     4f                              /* if not PF/GP/UD/DE, panic */
 289 
 290         orw     $CPU_DTRACE_DIVZERO, %cx
 291         movw    %cx, CPUC_DTRACE_FLAGS(%rax)
 292         jmp     2f
 293 
 294         /*
 295          * If we've taken a GPF, we don't (unfortunately) have the address that
 296          * induced the fault.  So instead of setting the fault to BADADDR,
 297          * we'll set the fault to ILLOP.
 298          */
 299 0:
 300         orw     $CPU_DTRACE_ILLOP, %cx
 301         movw    %cx, CPUC_DTRACE_FLAGS(%rax)
 302         jmp     2f
 303 1:
 304         orw     $CPU_DTRACE_BADADDR, %cx
 305         movw    %cx, CPUC_DTRACE_FLAGS(%rax)    /* set fault to bad addr */
 306         movq    %r15, CPUC_DTRACE_ILLVAL(%rax)
 307                                             /* fault addr is illegal value */
 308 2:
 309         movq    REGOFF_RIP(%rbp), %rdi
 310         movq    %rdi, %r12
 311         call    dtrace_instr_size
 312         addq    %rax, %r12
 313         movq    %r12, REGOFF_RIP(%rbp)
 314         INTR_POP
 315         call    x86_md_clear
 316         jmp     tr_iret_auto
 317         /*NOTREACHED*/
 318 3:
 319         leaq    dtrace_badflags(%rip), %rdi
 320         xorl    %eax, %eax
 321         call    panic
 322 4:
 323         leaq    dtrace_badtrap(%rip), %rdi
 324         xorl    %eax, %eax
 325         call    panic
 326         SET_SIZE(cmntrap_pushed)
 327         SET_SIZE(cmntrap)
 328         SET_SIZE(_cmntrap)
 329 
 330 /*
 331  * Declare a uintptr_t which has the size of _cmntrap to enable stack
 332  * traceback code to know when a regs structure is on the stack.
 333  */
 334         .globl  _cmntrap_size
 335         .align  CLONGSIZE
 336 _cmntrap_size:
 337         .NWORD  . - _cmntrap
 338         .type   _cmntrap_size, @object
 339 
 340 dtrace_badflags:
 341         .string "bad DTrace flags"
 342 
 343 dtrace_badtrap:
 344         .string "bad DTrace trap"
 345 
 346         .globl  trap            /* C handler called below */
 347 
 348         ENTRY_NP(cmninttrap)
 349 
 350         INTR_PUSH
 351         INTGATE_INIT_KERNEL_FLAGS
 352 
 353         TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
 354         TRACE_REGS(%rdi, %rsp, %rbx, %rcx)      /* Uses label 9 */
 355         TRACE_STAMP(%rdi)               /* Clobbers %eax, %edx, uses 9 */
 356 
 357         movq    %rsp, %rbp
 358 
 359         movl    %gs:CPU_ID, %edx
 360         xorl    %esi, %esi
 361         movq    %rsp, %rdi
 362         call    trap            /* trap(rp, addr, cpuid) handles all traps */
 363         jmp     _sys_rtt
 364         SET_SIZE(cmninttrap)
 365 
 366 #if !defined(__xpv)
 367         /*
 368          * Handle traps early in boot. Just revectors into C quickly as
 369          * these are always fatal errors.
 370          *
 371          * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap().
 372          */
 373         ENTRY(bop_trap_handler)
 374         movq    %rsp, %rdi
 375         sub     $8, %rsp
 376         call    bop_trap
 377         SET_SIZE(bop_trap_handler)
 378 #endif
 379 
 380         .globl  dtrace_user_probe
 381 
 382         ENTRY_NP(dtrace_trap)
 383 
 384         INTR_PUSH
 385 
 386         TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
 387         TRACE_REGS(%rdi, %rsp, %rbx, %rcx)      /* Uses label 9 */
 388         TRACE_STAMP(%rdi)               /* Clobbers %eax, %edx, uses 9 */
 389 
 390         movq    %rsp, %rbp
 391 
 392         movl    %gs:CPU_ID, %edx
 393 #if defined(__xpv)
 394         movq    %gs:CPU_VCPU_INFO, %rsi
 395         movq    VCPU_INFO_ARCH_CR2(%rsi), %rsi
 396 #else
 397         movq    %cr2, %rsi
 398 #endif
 399         movq    %rsp, %rdi
 400 
 401         ENABLE_INTR_FLAGS
 402 
 403         call    dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
 404         jmp     _sys_rtt
 405 
 406         SET_SIZE(dtrace_trap)
 407 
 408 /*
 409  * Return from _sys_trap routine.
 410  */
 411 
 412         ENTRY_NP(lwp_rtt_initial)
 413         movq    %gs:CPU_THREAD, %r15
 414         movq    T_STACK(%r15), %rsp     /* switch to the thread stack */
 415         movq    %rsp, %rbp
 416         call    __dtrace_probe___proc_start
 417         jmp     _lwp_rtt
 418 
 419         ENTRY_NP(lwp_rtt)
 420 
 421         /*
 422          * r14  lwp
 423          * rdx  lwp->lwp_procp
 424          * r15  curthread
 425          */
 426 
 427         movq    %gs:CPU_THREAD, %r15
 428         movq    T_STACK(%r15), %rsp     /* switch to the thread stack */
 429         movq    %rsp, %rbp
 430 _lwp_rtt:
 431         call    __dtrace_probe___proc_lwp__start
 432         movq    %gs:CPU_LWP, %r14
 433         movq    LWP_PROCP(%r14), %rdx
 434 
 435         /*
 436          * XX64 Is the stack misaligned correctly at this point?
 437          *      If not, we need to do a push before calling anything ..
 438          */
 439 
 440 #if defined(DEBUG)
 441         /*
 442          * If we were to run lwp_savectx at this point -without-
 443          * pcb_rupdate being set to 1, we'd end up sampling the hardware
 444          * state left by the previous running lwp, rather than setting
 445          * the values requested by the lwp creator.  Bad.
 446          */
 447         testb   $0x1, PCB_RUPDATE(%r14)
 448         jne     1f
 449         leaq    _no_pending_updates(%rip), %rdi
 450         movl    $__LINE__, %esi
 451         movq    %r14, %rdx
 452         xorl    %eax, %eax
 453         call    panic
 454 1:
 455 #endif
 456 
 457         /*
 458          * If agent lwp, clear %fs and %gs
 459          */
 460         cmpq    %r15, P_AGENTTP(%rdx)
 461         jne     1f
 462         xorl    %ecx, %ecx
 463         movq    %rcx, REGOFF_FS(%rsp)
 464         movq    %rcx, REGOFF_GS(%rsp)
 465         movw    %cx, LWP_PCB_FS(%r14)
 466         movw    %cx, LWP_PCB_GS(%r14)
 467 1:
 468         call    dtrace_systrace_rtt
 469         movq    REGOFF_RDX(%rsp), %rsi
 470         movq    REGOFF_RAX(%rsp), %rdi
 471         call    post_syscall            /* post_syscall(rval1, rval2) */
 472 
 473         /*
 474          * XXX - may want a fast path that avoids sys_rtt_common in the
 475          * most common case.
 476          */
 477         ALTENTRY(_sys_rtt)
 478         CLI(%rax)                       /* disable interrupts */
 479         ALTENTRY(_sys_rtt_ints_disabled)
 480         movq    %rsp, %rdi              /* pass rp to sys_rtt_common */
 481         call    sys_rtt_common          /* do common sys_rtt tasks */
 482         testq   %rax, %rax              /* returning to userland? */
 483         jz      sr_sup
 484 
 485         /*
 486          * Return to user
 487          */
 488         ASSERT_UPCALL_MASK_IS_SET
 489         cmpw    $UCS_SEL, REGOFF_CS(%rsp) /* test for native (64-bit) lwp? */
 490         je      sys_rtt_syscall
 491 
 492         /*
 493          * Return to 32-bit userland
 494          */
 495         ALTENTRY(sys_rtt_syscall32)
 496         USER32_POP
 497         call    x86_md_clear
 498         jmp     tr_iret_user
 499         /*NOTREACHED*/
 500 
 501         ALTENTRY(sys_rtt_syscall)
 502         /*
 503          * Return to 64-bit userland
 504          */
 505         USER_POP
 506         ALTENTRY(nopop_sys_rtt_syscall)
 507         call    x86_md_clear
 508         jmp     tr_iret_user
 509         /*NOTREACHED*/
 510         SET_SIZE(nopop_sys_rtt_syscall)
 511 
 512         /*
 513          * Return to supervisor
 514          * NOTE: to make the check in trap() that tests if we are executing
 515          * segment register fixup/restore code work properly, sr_sup MUST be
 516          * after _sys_rtt .
 517          */
 518         ALTENTRY(sr_sup)
 519         /*
 520          * Restore regs before doing iretq to kernel mode
 521          */
 522         INTR_POP
 523         jmp     tr_iret_kernel
 524         .globl  _sys_rtt_end
 525 _sys_rtt_end:
 526         /*NOTREACHED*/
 527         SET_SIZE(sr_sup)
 528         SET_SIZE(_sys_rtt_end)
 529         SET_SIZE(lwp_rtt)
 530         SET_SIZE(lwp_rtt_initial)
 531         SET_SIZE(_sys_rtt_ints_disabled)
 532         SET_SIZE(_sys_rtt)
 533         SET_SIZE(sys_rtt_syscall)
 534         SET_SIZE(sys_rtt_syscall32)
 535 
 536         /*
 537          * XX64 quick and dirty port from the i386 version. Since we
 538          * believe the amd64 tsc is more reliable, could this code be
 539          * simpler?
 540          */
 541         ENTRY_NP(freq_tsc)
 542         pushq   %rbp
 543         movq    %rsp, %rbp
 544         movq    %rdi, %r9       /* save pit_counter */
 545         pushq   %rbx
 546 
 547 / We have a TSC, but we have no way in general to know how reliable it is.
 548 / Usually a marginal TSC behaves appropriately unless not enough time
 549 / elapses between reads. A reliable TSC can be read as often and as rapidly
 550 / as desired. The simplistic approach of reading the TSC counter and
 551 / correlating to the PIT counter cannot be naively followed. Instead estimates
 552 / have to be taken to successively refine a guess at the speed of the cpu
 553 / and then the TSC and PIT counter are correlated. In practice very rarely
 554 / is more than one quick loop required for an estimate. Measures have to be
 555 / taken to prevent the PIT counter from wrapping beyond its resolution and for
 556 / measuring the clock rate of very fast processors.
 557 /
 558 / The following constant can be tuned. It should be such that the loop does
 559 / not take too many nor too few PIT counts to execute. If this value is too
 560 / large, then on slow machines the loop will take a long time, or the PIT
 561 / counter may even wrap. If this value is too small, then on fast machines
 562 / the PIT counter may count so few ticks that the resolution of the PIT
 563 / itself causes a bad guess. Because this code is used in machines with
 564 / marginal TSC's and/or IO, if this value is too small on those, it may
 565 / cause the calculated cpu frequency to vary slightly from boot to boot.
 566 /
 567 / In all cases even if this constant is set inappropriately, the algorithm
 568 / will still work and the caller should be able to handle variances in the
 569 / calculation of cpu frequency, but the calculation will be inefficient and
 570 / take a disproportionate amount of time relative to a well selected value.
 571 / As the slowest supported cpu becomes faster, this constant should be
 572 / carefully increased.
 573 
 574         movl    $0x8000, %ecx
 575 
 576         / to make sure the instruction cache has been warmed
 577         clc
 578 
 579         jmp     freq_tsc_loop
 580 
 581 / The following block of code up to and including the latching of the PIT
 582 / counter after freq_tsc_perf_loop is very critical and very carefully
 583 / written, it should only be modified with great care. freq_tsc_loop to
 584 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
 585 / freq_tsc_perf_loop up to the unlatching of the PIT counter.
 586 
 587         .align  32
 588 freq_tsc_loop:
 589         / save the loop count in %ebx
 590         movl    %ecx, %ebx
 591 
 592         / initialize the PIT counter and start a count down
 593         movb    $PIT_LOADMODE, %al
 594         outb    $PITCTL_PORT
 595         movb    $0xff, %al
 596         outb    $PITCTR0_PORT
 597         outb    $PITCTR0_PORT
 598 
 599         / read the TSC and store the TS in %edi:%esi
 600         rdtsc
 601         movl    %eax, %esi
 602 
 603 freq_tsc_perf_loop:
 604         movl    %edx, %edi
 605         movl    %eax, %esi
 606         movl    %edx, %edi
 607         loop    freq_tsc_perf_loop
 608 
 609         / read the TSC and store the LSW in %ecx
 610         rdtsc
 611         movl    %eax, %ecx
 612 
 613         / latch the PIT counter and status
 614         movb    $_CONST(PIT_READBACK|PIT_READBACKC0), %al
 615         outb    $PITCTL_PORT
 616 
 617         / remember if the icache has been warmed
 618         setc    %ah
 619 
 620         / read the PIT status
 621         inb     $PITCTR0_PORT
 622         shll    $8, %eax
 623 
 624         / read PIT count
 625         inb     $PITCTR0_PORT
 626         shll    $8, %eax
 627         inb     $PITCTR0_PORT
 628         bswap   %eax
 629 
 630         / check to see if the PIT count was loaded into the CE
 631         btw     $_CONST(PITSTAT_NULLCNT+8), %ax
 632         jc      freq_tsc_increase_count
 633 
 634         / check to see if PIT counter wrapped
 635         btw     $_CONST(PITSTAT_OUTPUT+8), %ax
 636         jnc     freq_tsc_pit_did_not_wrap
 637 
 638         / halve count
 639         shrl    $1, %ebx
 640         movl    %ebx, %ecx
 641 
 642         / the instruction cache has been warmed
 643         stc
 644 
 645         jmp     freq_tsc_loop
 646 
 647 freq_tsc_increase_count:
 648         shll    $1, %ebx
 649         jc      freq_tsc_too_fast
 650 
 651         movl    %ebx, %ecx
 652 
 653         / the instruction cache has been warmed
 654         stc
 655 
 656         jmp     freq_tsc_loop
 657 
 658 freq_tsc_pit_did_not_wrap:
 659         roll    $16, %eax
 660 
 661         cmpw    $0x2000, %ax
 662         notw    %ax
 663         jb      freq_tsc_sufficient_duration
 664 
 665 freq_tsc_calculate:
 666         / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
 667         / then on the second CLK pulse the CE is decremented, therefore mode 0
 668         / is really a (count + 1) counter, ugh
 669         xorl    %esi, %esi
 670         movw    %ax, %si
 671         incl    %esi
 672 
 673         movl    $0xf000, %eax
 674         mull    %ebx
 675 
 676         / tuck away (target_pit_count * loop_count)
 677         movl    %edx, %ecx
 678         movl    %eax, %ebx
 679 
 680         movl    %esi, %eax
 681         movl    $0xffffffff, %edx
 682         mull    %edx
 683 
 684         addl    %esi, %eax
 685         adcl    $0, %edx
 686 
 687         cmpl    %ecx, %edx
 688         ja      freq_tsc_div_safe
 689         jb      freq_tsc_too_fast
 690 
 691         cmpl    %ebx, %eax
 692         jbe     freq_tsc_too_fast
 693 
 694 freq_tsc_div_safe:
 695         movl    %ecx, %edx
 696         movl    %ebx, %eax
 697 
 698         movl    %esi, %ecx
 699         divl    %ecx
 700 
 701         movl    %eax, %ecx
 702 
 703         / the instruction cache has been warmed
 704         stc
 705 
 706         jmp     freq_tsc_loop
 707 
 708 freq_tsc_sufficient_duration:
 709         / test to see if the icache has been warmed
 710         btl     $16, %eax
 711         jnc     freq_tsc_calculate
 712 
 713         / recall mode 0 is a (count + 1) counter
 714         andl    $0xffff, %eax
 715         incl    %eax
 716 
 717         / save the number of PIT counts
 718         movl    %eax, (%r9)
 719 
 720         / calculate the number of TS's that elapsed
 721         movl    %ecx, %eax
 722         subl    %esi, %eax
 723         sbbl    %edi, %edx
 724 
 725         jmp     freq_tsc_end
 726 
 727 freq_tsc_too_fast:
 728         / return 0 as a 64 bit quantity
 729         xorl    %eax, %eax
 730         xorl    %edx, %edx
 731 
 732 freq_tsc_end:
 733         shlq    $32, %rdx
 734         orq     %rdx, %rax
 735 
 736         popq    %rbx
 737         leaveq
 738         ret
 739         SET_SIZE(freq_tsc)
 740