1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 /* 26 * Copyright 2019 Joyent, Inc. 27 */ 28 29 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ 30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ 31 /* All Rights Reserved */ 32 33 /* Copyright (c) 1987, 1988 Microsoft Corporation */ 34 /* All Rights Reserved */ 35 36 37 #include <sys/asm_linkage.h> 38 #include <sys/asm_misc.h> 39 #include <sys/regset.h> 40 #include <sys/privregs.h> 41 #include <sys/psw.h> 42 #include <sys/reboot.h> 43 #include <sys/x86_archext.h> 44 #include <sys/machparam.h> 45 46 #include <sys/segments.h> 47 #include <sys/pcb.h> 48 #include <sys/trap.h> 49 #include <sys/ftrace.h> 50 #include <sys/traptrace.h> 51 #include <sys/clock.h> 52 #include <sys/cmn_err.h> 53 #include <sys/pit.h> 54 #include <sys/panic.h> 55 56 #if defined(__xpv) 57 #include <sys/hypervisor.h> 58 #endif 59 60 #include "assym.h" 61 62 /* 63 * Our assumptions: 64 * - We are running in protected-paged mode. 65 * - Interrupts are disabled. 66 * - The GDT and IDT are the callers; we need our copies. 67 * - The kernel's text, initialized data and bss are mapped. 68 * 69 * Our actions: 70 * - Save arguments 71 * - Initialize our stack pointer to the thread 0 stack (t0stack) 72 * and leave room for a phony "struct regs". 73 * - Our GDT and IDT need to get munged. 74 * - Since we are using the boot's GDT descriptors, we need 75 * to copy them into our GDT before we switch to ours. 76 * - We start using our GDT by loading correct values in the 77 * selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL, 78 * gs=KGS_SEL). 79 * - The default LDT entry for syscall is set. 80 * - We load the default LDT into the hardware LDT register. 81 * - We load the default TSS into the hardware task register. 82 * - Check for cpu type, i.e. 486 vs. P5 vs. P6 etc. 83 * - mlsetup(%esp) gets called. 84 * - We change our appearance to look like the real thread 0. 85 * (NOTE: making ourselves to be a real thread may be a noop) 86 * - main() gets called. (NOTE: main() never returns). 87 * 88 * NOW, the real code! 89 */ 90 /* 91 * The very first thing in the kernel's text segment must be a jump 92 * to the os/fakebop.c startup code. 93 */ 94 .text 95 jmp _start 96 97 /* 98 * Globals: 99 */ 100 .globl _locore_start 101 .globl mlsetup 102 .globl main 103 .globl panic 104 .globl t0stack 105 .globl t0 106 .globl sysp 107 .globl edata 108 109 /* 110 * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h) 111 */ 112 .globl bootops 113 .globl bootopsp 114 115 /* 116 * NOTE: t0stack should be the first thing in the data section so that 117 * if it ever overflows, it will fault on the last kernel text page. 118 */ 119 .data 120 .comm t0stack, DEFAULTSTKSZ, 32 121 .comm t0, 4094, 32 122 123 124 /* 125 * kobj_init() vectors us back to here with (note) a slightly different 126 * set of arguments than _start is given (see lint prototypes above). 127 * 128 * XXX Make this less vile, please. 129 */ 130 ENTRY_NP(_locore_start) 131 132 /* 133 * %rdi = boot services (should die someday) 134 * %rdx = bootops 135 * end 136 */ 137 138 leaq edata(%rip), %rbp /* reference edata for ksyms */ 139 movq $0, (%rbp) /* limit stack back trace */ 140 141 /* 142 * Initialize our stack pointer to the thread 0 stack (t0stack) 143 * and leave room for a "struct regs" for lwp0. Note that the 144 * stack doesn't actually align to a 16-byte boundary until just 145 * before we call mlsetup because we want to use %rsp to point at 146 * our regs structure. 147 */ 148 leaq t0stack(%rip), %rsp 149 addq $_CONST(DEFAULTSTKSZ - REGSIZE), %rsp 150 #if (REGSIZE & 15) == 0 151 subq $8, %rsp 152 #endif 153 /* 154 * Save call back for special x86 boot services vector 155 */ 156 movq %rdi, sysp(%rip) 157 158 movq %rdx, bootops(%rip) /* save bootops */ 159 movq $bootops, bootopsp(%rip) 160 161 /* 162 * Save arguments and flags, if only for debugging .. 163 */ 164 movq %rdi, REGOFF_RDI(%rsp) 165 movq %rsi, REGOFF_RSI(%rsp) 166 movq %rdx, REGOFF_RDX(%rsp) 167 movq %rcx, REGOFF_RCX(%rsp) 168 movq %r8, REGOFF_R8(%rsp) 169 movq %r9, REGOFF_R9(%rsp) 170 pushf 171 popq %r11 172 movq %r11, REGOFF_RFL(%rsp) 173 174 #if !defined(__xpv) 175 /* 176 * Enable write protect and alignment check faults. 177 */ 178 movq %cr0, %rax 179 orq $_CONST(CR0_WP|CR0_AM), %rax 180 andq $_BITNOT(CR0_WT|CR0_CE), %rax 181 movq %rax, %cr0 182 #endif /* __xpv */ 183 184 /* 185 * (We just assert this works by virtue of being here) 186 */ 187 bts $X86FSET_CPUID, x86_featureset(%rip) 188 189 /* 190 * mlsetup() gets called with a struct regs as argument, while 191 * main takes no args and should never return. 192 */ 193 xorl %ebp, %ebp 194 movq %rsp, %rdi 195 pushq %rbp 196 /* (stack pointer now aligned on 16-byte boundary right here) */ 197 movq %rsp, %rbp 198 call mlsetup 199 call main 200 /* NOTREACHED */ 201 leaq __return_from_main(%rip), %rdi 202 xorl %eax, %eax 203 call panic 204 SET_SIZE(_locore_start) 205 206 __return_from_main: 207 .string "main() returned" 208 __unsupported_cpu: 209 .string "486 style cpu detected - no longer supported!" 210 211 #if defined(DEBUG) 212 _no_pending_updates: 213 .string "locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1" 214 #endif 215 216 /* 217 * For stack layout, see privregs.h 218 * When cmntrap gets called, the error code and trap number have been pushed. 219 * When cmntrap_pushed gets called, the entire struct regs has been pushed. 220 */ 221 222 .globl trap /* C handler called below */ 223 224 ENTRY_NP2(cmntrap, _cmntrap) 225 226 INTR_PUSH 227 228 ALTENTRY(cmntrap_pushed) 229 230 movq %rsp, %rbp 231 232 /* 233 * - if this is a #pf i.e. T_PGFLT, %r15 is live 234 * and contains the faulting address i.e. a copy of %cr2 235 * 236 * - if this is a #db i.e. T_SGLSTP, %r15 is live 237 * and contains the value of %db6 238 */ 239 240 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */ 241 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */ 242 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */ 243 244 /* 245 * We must first check if DTrace has set its NOFAULT bit. This 246 * regrettably must happen before the trap stack is recorded, because 247 * this requires a call to getpcstack() and may induce recursion if an 248 * fbt::getpcstack: enabling is inducing the bad load. 249 */ 250 movl %gs:CPU_ID, %eax 251 shlq $CPU_CORE_SHIFT, %rax 252 leaq cpu_core(%rip), %r8 253 addq %r8, %rax 254 movw CPUC_DTRACE_FLAGS(%rax), %cx 255 testw $CPU_DTRACE_NOFAULT, %cx 256 jnz .dtrace_induced 257 258 TRACE_STACK(%rdi) 259 260 movq %rbp, %rdi 261 movq %r15, %rsi 262 movl %gs:CPU_ID, %edx 263 264 /* 265 * We know that this isn't a DTrace non-faulting load; we can now safely 266 * reenable interrupts. (In the case of pagefaults, we enter through an 267 * interrupt gate.) 268 */ 269 ENABLE_INTR_FLAGS 270 271 call trap /* trap(rp, addr, cpuid) handles all traps */ 272 jmp _sys_rtt 273 274 .dtrace_induced: 275 cmpw $KCS_SEL, REGOFF_CS(%rbp) /* test CS for user-mode trap */ 276 jne 3f /* if from user, panic */ 277 278 cmpl $T_PGFLT, REGOFF_TRAPNO(%rbp) 279 je 1f 280 281 cmpl $T_GPFLT, REGOFF_TRAPNO(%rbp) 282 je 0f 283 284 cmpl $T_ILLINST, REGOFF_TRAPNO(%rbp) 285 je 0f 286 287 cmpl $T_ZERODIV, REGOFF_TRAPNO(%rbp) 288 jne 4f /* if not PF/GP/UD/DE, panic */ 289 290 orw $CPU_DTRACE_DIVZERO, %cx 291 movw %cx, CPUC_DTRACE_FLAGS(%rax) 292 jmp 2f 293 294 /* 295 * If we've taken a GPF, we don't (unfortunately) have the address that 296 * induced the fault. So instead of setting the fault to BADADDR, 297 * we'll set the fault to ILLOP. 298 */ 299 0: 300 orw $CPU_DTRACE_ILLOP, %cx 301 movw %cx, CPUC_DTRACE_FLAGS(%rax) 302 jmp 2f 303 1: 304 orw $CPU_DTRACE_BADADDR, %cx 305 movw %cx, CPUC_DTRACE_FLAGS(%rax) /* set fault to bad addr */ 306 movq %r15, CPUC_DTRACE_ILLVAL(%rax) 307 /* fault addr is illegal value */ 308 2: 309 movq REGOFF_RIP(%rbp), %rdi 310 movq %rdi, %r12 311 call dtrace_instr_size 312 addq %rax, %r12 313 movq %r12, REGOFF_RIP(%rbp) 314 INTR_POP 315 call x86_md_clear 316 jmp tr_iret_auto 317 /*NOTREACHED*/ 318 3: 319 leaq dtrace_badflags(%rip), %rdi 320 xorl %eax, %eax 321 call panic 322 4: 323 leaq dtrace_badtrap(%rip), %rdi 324 xorl %eax, %eax 325 call panic 326 SET_SIZE(cmntrap_pushed) 327 SET_SIZE(cmntrap) 328 SET_SIZE(_cmntrap) 329 330 /* 331 * Declare a uintptr_t which has the size of _cmntrap to enable stack 332 * traceback code to know when a regs structure is on the stack. 333 */ 334 .globl _cmntrap_size 335 .align CLONGSIZE 336 _cmntrap_size: 337 .NWORD . - _cmntrap 338 .type _cmntrap_size, @object 339 340 dtrace_badflags: 341 .string "bad DTrace flags" 342 343 dtrace_badtrap: 344 .string "bad DTrace trap" 345 346 .globl trap /* C handler called below */ 347 348 ENTRY_NP(cmninttrap) 349 350 INTR_PUSH 351 INTGATE_INIT_KERNEL_FLAGS 352 353 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */ 354 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */ 355 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */ 356 357 movq %rsp, %rbp 358 359 movl %gs:CPU_ID, %edx 360 xorl %esi, %esi 361 movq %rsp, %rdi 362 call trap /* trap(rp, addr, cpuid) handles all traps */ 363 jmp _sys_rtt 364 SET_SIZE(cmninttrap) 365 366 #if !defined(__xpv) 367 /* 368 * Handle traps early in boot. Just revectors into C quickly as 369 * these are always fatal errors. 370 * 371 * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap(). 372 */ 373 ENTRY(bop_trap_handler) 374 movq %rsp, %rdi 375 sub $8, %rsp 376 call bop_trap 377 SET_SIZE(bop_trap_handler) 378 #endif 379 380 .globl dtrace_user_probe 381 382 ENTRY_NP(dtrace_trap) 383 384 INTR_PUSH 385 386 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */ 387 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */ 388 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */ 389 390 movq %rsp, %rbp 391 392 movl %gs:CPU_ID, %edx 393 #if defined(__xpv) 394 movq %gs:CPU_VCPU_INFO, %rsi 395 movq VCPU_INFO_ARCH_CR2(%rsi), %rsi 396 #else 397 movq %cr2, %rsi 398 #endif 399 movq %rsp, %rdi 400 401 ENABLE_INTR_FLAGS 402 403 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */ 404 jmp _sys_rtt 405 406 SET_SIZE(dtrace_trap) 407 408 /* 409 * Return from _sys_trap routine. 410 */ 411 412 ENTRY_NP(lwp_rtt_initial) 413 movq %gs:CPU_THREAD, %r15 414 movq T_STACK(%r15), %rsp /* switch to the thread stack */ 415 movq %rsp, %rbp 416 call __dtrace_probe___proc_start 417 jmp _lwp_rtt 418 419 ENTRY_NP(lwp_rtt) 420 421 /* 422 * r14 lwp 423 * rdx lwp->lwp_procp 424 * r15 curthread 425 */ 426 427 movq %gs:CPU_THREAD, %r15 428 movq T_STACK(%r15), %rsp /* switch to the thread stack */ 429 movq %rsp, %rbp 430 _lwp_rtt: 431 call __dtrace_probe___proc_lwp__start 432 movq %gs:CPU_LWP, %r14 433 movq LWP_PROCP(%r14), %rdx 434 435 /* 436 * XX64 Is the stack misaligned correctly at this point? 437 * If not, we need to do a push before calling anything .. 438 */ 439 440 #if defined(DEBUG) 441 /* 442 * If we were to run lwp_savectx at this point -without- 443 * pcb_rupdate being set to 1, we'd end up sampling the hardware 444 * state left by the previous running lwp, rather than setting 445 * the values requested by the lwp creator. Bad. 446 */ 447 testb $0x1, PCB_RUPDATE(%r14) 448 jne 1f 449 leaq _no_pending_updates(%rip), %rdi 450 movl $__LINE__, %esi 451 movq %r14, %rdx 452 xorl %eax, %eax 453 call panic 454 1: 455 #endif 456 457 /* 458 * If agent lwp, clear %fs and %gs 459 */ 460 cmpq %r15, P_AGENTTP(%rdx) 461 jne 1f 462 xorl %ecx, %ecx 463 movq %rcx, REGOFF_FS(%rsp) 464 movq %rcx, REGOFF_GS(%rsp) 465 movw %cx, LWP_PCB_FS(%r14) 466 movw %cx, LWP_PCB_GS(%r14) 467 1: 468 call dtrace_systrace_rtt 469 movq REGOFF_RDX(%rsp), %rsi 470 movq REGOFF_RAX(%rsp), %rdi 471 call post_syscall /* post_syscall(rval1, rval2) */ 472 473 /* 474 * XXX - may want a fast path that avoids sys_rtt_common in the 475 * most common case. 476 */ 477 ALTENTRY(_sys_rtt) 478 CLI(%rax) /* disable interrupts */ 479 ALTENTRY(_sys_rtt_ints_disabled) 480 movq %rsp, %rdi /* pass rp to sys_rtt_common */ 481 call sys_rtt_common /* do common sys_rtt tasks */ 482 testq %rax, %rax /* returning to userland? */ 483 jz sr_sup 484 485 /* 486 * Return to user 487 */ 488 ASSERT_UPCALL_MASK_IS_SET 489 cmpw $UCS_SEL, REGOFF_CS(%rsp) /* test for native (64-bit) lwp? */ 490 je sys_rtt_syscall 491 492 /* 493 * Return to 32-bit userland 494 */ 495 ALTENTRY(sys_rtt_syscall32) 496 USER32_POP 497 call x86_md_clear 498 jmp tr_iret_user 499 /*NOTREACHED*/ 500 501 ALTENTRY(sys_rtt_syscall) 502 /* 503 * Return to 64-bit userland 504 */ 505 USER_POP 506 ALTENTRY(nopop_sys_rtt_syscall) 507 call x86_md_clear 508 jmp tr_iret_user 509 /*NOTREACHED*/ 510 SET_SIZE(nopop_sys_rtt_syscall) 511 512 /* 513 * Return to supervisor 514 * NOTE: to make the check in trap() that tests if we are executing 515 * segment register fixup/restore code work properly, sr_sup MUST be 516 * after _sys_rtt . 517 */ 518 ALTENTRY(sr_sup) 519 /* 520 * Restore regs before doing iretq to kernel mode 521 */ 522 INTR_POP 523 jmp tr_iret_kernel 524 .globl _sys_rtt_end 525 _sys_rtt_end: 526 /*NOTREACHED*/ 527 SET_SIZE(sr_sup) 528 SET_SIZE(_sys_rtt_end) 529 SET_SIZE(lwp_rtt) 530 SET_SIZE(lwp_rtt_initial) 531 SET_SIZE(_sys_rtt_ints_disabled) 532 SET_SIZE(_sys_rtt) 533 SET_SIZE(sys_rtt_syscall) 534 SET_SIZE(sys_rtt_syscall32) 535 536 /* 537 * XX64 quick and dirty port from the i386 version. Since we 538 * believe the amd64 tsc is more reliable, could this code be 539 * simpler? 540 */ 541 ENTRY_NP(freq_tsc) 542 pushq %rbp 543 movq %rsp, %rbp 544 movq %rdi, %r9 /* save pit_counter */ 545 pushq %rbx 546 547 / We have a TSC, but we have no way in general to know how reliable it is. 548 / Usually a marginal TSC behaves appropriately unless not enough time 549 / elapses between reads. A reliable TSC can be read as often and as rapidly 550 / as desired. The simplistic approach of reading the TSC counter and 551 / correlating to the PIT counter cannot be naively followed. Instead estimates 552 / have to be taken to successively refine a guess at the speed of the cpu 553 / and then the TSC and PIT counter are correlated. In practice very rarely 554 / is more than one quick loop required for an estimate. Measures have to be 555 / taken to prevent the PIT counter from wrapping beyond its resolution and for 556 / measuring the clock rate of very fast processors. 557 / 558 / The following constant can be tuned. It should be such that the loop does 559 / not take too many nor too few PIT counts to execute. If this value is too 560 / large, then on slow machines the loop will take a long time, or the PIT 561 / counter may even wrap. If this value is too small, then on fast machines 562 / the PIT counter may count so few ticks that the resolution of the PIT 563 / itself causes a bad guess. Because this code is used in machines with 564 / marginal TSC's and/or IO, if this value is too small on those, it may 565 / cause the calculated cpu frequency to vary slightly from boot to boot. 566 / 567 / In all cases even if this constant is set inappropriately, the algorithm 568 / will still work and the caller should be able to handle variances in the 569 / calculation of cpu frequency, but the calculation will be inefficient and 570 / take a disproportionate amount of time relative to a well selected value. 571 / As the slowest supported cpu becomes faster, this constant should be 572 / carefully increased. 573 574 movl $0x8000, %ecx 575 576 / to make sure the instruction cache has been warmed 577 clc 578 579 jmp freq_tsc_loop 580 581 / The following block of code up to and including the latching of the PIT 582 / counter after freq_tsc_perf_loop is very critical and very carefully 583 / written, it should only be modified with great care. freq_tsc_loop to 584 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in 585 / freq_tsc_perf_loop up to the unlatching of the PIT counter. 586 587 .align 32 588 freq_tsc_loop: 589 / save the loop count in %ebx 590 movl %ecx, %ebx 591 592 / initialize the PIT counter and start a count down 593 movb $PIT_LOADMODE, %al 594 outb $PITCTL_PORT 595 movb $0xff, %al 596 outb $PITCTR0_PORT 597 outb $PITCTR0_PORT 598 599 / read the TSC and store the TS in %edi:%esi 600 rdtsc 601 movl %eax, %esi 602 603 freq_tsc_perf_loop: 604 movl %edx, %edi 605 movl %eax, %esi 606 movl %edx, %edi 607 loop freq_tsc_perf_loop 608 609 / read the TSC and store the LSW in %ecx 610 rdtsc 611 movl %eax, %ecx 612 613 / latch the PIT counter and status 614 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al 615 outb $PITCTL_PORT 616 617 / remember if the icache has been warmed 618 setc %ah 619 620 / read the PIT status 621 inb $PITCTR0_PORT 622 shll $8, %eax 623 624 / read PIT count 625 inb $PITCTR0_PORT 626 shll $8, %eax 627 inb $PITCTR0_PORT 628 bswap %eax 629 630 / check to see if the PIT count was loaded into the CE 631 btw $_CONST(PITSTAT_NULLCNT+8), %ax 632 jc freq_tsc_increase_count 633 634 / check to see if PIT counter wrapped 635 btw $_CONST(PITSTAT_OUTPUT+8), %ax 636 jnc freq_tsc_pit_did_not_wrap 637 638 / halve count 639 shrl $1, %ebx 640 movl %ebx, %ecx 641 642 / the instruction cache has been warmed 643 stc 644 645 jmp freq_tsc_loop 646 647 freq_tsc_increase_count: 648 shll $1, %ebx 649 jc freq_tsc_too_fast 650 651 movl %ebx, %ecx 652 653 / the instruction cache has been warmed 654 stc 655 656 jmp freq_tsc_loop 657 658 freq_tsc_pit_did_not_wrap: 659 roll $16, %eax 660 661 cmpw $0x2000, %ax 662 notw %ax 663 jb freq_tsc_sufficient_duration 664 665 freq_tsc_calculate: 666 / in mode 0, the PIT loads the count into the CE on the first CLK pulse, 667 / then on the second CLK pulse the CE is decremented, therefore mode 0 668 / is really a (count + 1) counter, ugh 669 xorl %esi, %esi 670 movw %ax, %si 671 incl %esi 672 673 movl $0xf000, %eax 674 mull %ebx 675 676 / tuck away (target_pit_count * loop_count) 677 movl %edx, %ecx 678 movl %eax, %ebx 679 680 movl %esi, %eax 681 movl $0xffffffff, %edx 682 mull %edx 683 684 addl %esi, %eax 685 adcl $0, %edx 686 687 cmpl %ecx, %edx 688 ja freq_tsc_div_safe 689 jb freq_tsc_too_fast 690 691 cmpl %ebx, %eax 692 jbe freq_tsc_too_fast 693 694 freq_tsc_div_safe: 695 movl %ecx, %edx 696 movl %ebx, %eax 697 698 movl %esi, %ecx 699 divl %ecx 700 701 movl %eax, %ecx 702 703 / the instruction cache has been warmed 704 stc 705 706 jmp freq_tsc_loop 707 708 freq_tsc_sufficient_duration: 709 / test to see if the icache has been warmed 710 btl $16, %eax 711 jnc freq_tsc_calculate 712 713 / recall mode 0 is a (count + 1) counter 714 andl $0xffff, %eax 715 incl %eax 716 717 / save the number of PIT counts 718 movl %eax, (%r9) 719 720 / calculate the number of TS's that elapsed 721 movl %ecx, %eax 722 subl %esi, %eax 723 sbbl %edi, %edx 724 725 jmp freq_tsc_end 726 727 freq_tsc_too_fast: 728 / return 0 as a 64 bit quantity 729 xorl %eax, %eax 730 xorl %edx, %edx 731 732 freq_tsc_end: 733 shlq $32, %rdx 734 orq %rdx, %rax 735 736 popq %rbx 737 leaveq 738 ret 739 SET_SIZE(freq_tsc) 740