8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  25  * Copyright (c) 2014 by Delphix. All rights reserved.
  26  * Copyright 2018 Joyent, Inc.
  27  */
  28 
  29 /*
  30  *  Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
  31  *  Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
  32  *    All Rights Reserved
  33  */
  34 
  35 /*
  36  * Copyright (c) 2009, Intel Corporation.
  37  * All rights reserved.
  38  */
  39 
  40 /*
  41  * General assembly language routines.
  42  * It is the intent of this file to contain routines that are
  43  * independent of the specific kernel architecture, and those that are
  44  * common across kernel architectures.
  45  * As architectures diverge, and implementations of specific
  46  * architecture-dependent routines change, the routines should be moved
  47  * from this file into the respective ../`arch -k`/subr.s file.
  48  */
  49 
  50 #include <sys/asm_linkage.h>
  51 #include <sys/asm_misc.h>
  52 #include <sys/panic.h>
  53 #include <sys/ontrap.h>
  54 #include <sys/regset.h>
  55 #include <sys/privregs.h>
  56 #include <sys/reboot.h>
  57 #include <sys/psw.h>
  58 #include <sys/x86_archext.h>
  59 
  60 #if defined(__lint)
  61 #include <sys/types.h>
  62 #include <sys/systm.h>
  63 #include <sys/thread.h>
  64 #include <sys/archsystm.h>
  65 #include <sys/byteorder.h>
  66 #include <sys/dtrace.h>
  67 #include <sys/ftrace.h>
  68 #else   /* __lint */
  69 #include "assym.h"
  70 #endif  /* __lint */
  71 #include <sys/dditypes.h>
  72 
  73 /*
  74  * on_fault()
  75  *
  76  * Catch lofault faults. Like setjmp except it returns one
  77  * if code following causes uncorrectable fault. Turned off
  78  * by calling no_fault(). Note that while under on_fault(),
  79  * SMAP is disabled. For more information see
  80  * uts/intel/ia32/ml/copy.s.
  81  */
  82 
  83 #if defined(__lint)
  84 
  85 /* ARGSUSED */
  86 int
  87 on_fault(label_t *ljb)
  88 { return (0); }
  89 
  90 void
  91 no_fault(void)
  92 {}
  93 
  94 #else   /* __lint */
  95 
  96 #if defined(__amd64)
  97 
  98         ENTRY(on_fault)
  99         movq    %gs:CPU_THREAD, %rsi
 100         leaq    catch_fault(%rip), %rdx
 101         movq    %rdi, T_ONFAULT(%rsi)           /* jumpbuf in t_onfault */
 102         movq    %rdx, T_LOFAULT(%rsi)           /* catch_fault in t_lofault */
 103         call    smap_disable                    /* allow user accesses */
 104         jmp     setjmp                          /* let setjmp do the rest */
 105 
 106 catch_fault:
 107         movq    %gs:CPU_THREAD, %rsi
 108         movq    T_ONFAULT(%rsi), %rdi           /* address of save area */
 109         xorl    %eax, %eax
 110         movq    %rax, T_ONFAULT(%rsi)           /* turn off onfault */
 111         movq    %rax, T_LOFAULT(%rsi)           /* turn off lofault */
 112         call    smap_enable                     /* disallow user accesses */
 113         jmp     longjmp                         /* let longjmp do the rest */
 114         SET_SIZE(on_fault)
 115 
 116         ENTRY(no_fault)
 117         movq    %gs:CPU_THREAD, %rsi
 118         xorl    %eax, %eax
 119         movq    %rax, T_ONFAULT(%rsi)           /* turn off onfault */
 120         movq    %rax, T_LOFAULT(%rsi)           /* turn off lofault */
 121         call    smap_enable                     /* disallow user accesses */
 122         ret
 123         SET_SIZE(no_fault)
 124 
 125 #elif defined(__i386)
 126 
 127         ENTRY(on_fault)
 128         movl    %gs:CPU_THREAD, %edx
 129         movl    4(%esp), %eax                   /* jumpbuf address */
 130         leal    catch_fault, %ecx
 131         movl    %eax, T_ONFAULT(%edx)           /* jumpbuf in t_onfault */
 132         movl    %ecx, T_LOFAULT(%edx)           /* catch_fault in t_lofault */
 133         jmp     setjmp                          /* let setjmp do the rest */
 134 
 135 catch_fault:
 136         movl    %gs:CPU_THREAD, %edx
 137         xorl    %eax, %eax
 138         movl    T_ONFAULT(%edx), %ecx           /* address of save area */
 139         movl    %eax, T_ONFAULT(%edx)           /* turn off onfault */
 140         movl    %eax, T_LOFAULT(%edx)           /* turn off lofault */
 141         pushl   %ecx
 142         call    longjmp                         /* let longjmp do the rest */
 143         SET_SIZE(on_fault)
 144 
 145         ENTRY(no_fault)
 146         movl    %gs:CPU_THREAD, %edx
 147         xorl    %eax, %eax
 148         movl    %eax, T_ONFAULT(%edx)           /* turn off onfault */
 149         movl    %eax, T_LOFAULT(%edx)           /* turn off lofault */
 150         ret
 151         SET_SIZE(no_fault)
 152 
 153 #endif  /* __i386 */
 154 #endif  /* __lint */
 155 
 156 /*
 157  * Default trampoline code for on_trap() (see <sys/ontrap.h>).  We just
 158  * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
 159  */
 160 
 161 #if defined(lint)
 162 
 163 void
 164 on_trap_trampoline(void)
 165 {}
 166 
 167 #else   /* __lint */
 168 
 169 #if defined(__amd64)
 170 
 171         ENTRY(on_trap_trampoline)
 172         movq    %gs:CPU_THREAD, %rsi
 173         movq    T_ONTRAP(%rsi), %rdi
 174         addq    $OT_JMPBUF, %rdi
 175         jmp     longjmp
 176         SET_SIZE(on_trap_trampoline)
 177 
 178 #elif defined(__i386)
 179 
 180         ENTRY(on_trap_trampoline)
 181         movl    %gs:CPU_THREAD, %eax
 182         movl    T_ONTRAP(%eax), %eax
 183         addl    $OT_JMPBUF, %eax
 184         pushl   %eax
 185         call    longjmp
 186         SET_SIZE(on_trap_trampoline)
 187 
 188 #endif  /* __i386 */
 189 #endif  /* __lint */
 190 
 191 /*
 192  * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
 193  * more information about the on_trap() mechanism.  If the on_trap_data is the
 194  * same as the topmost stack element, we just modify that element.
 195  */
 196 #if defined(lint)
 197 
 198 /*ARGSUSED*/
 199 int
 200 on_trap(on_trap_data_t *otp, uint_t prot)
 201 { return (0); }
 202 
 203 #else   /* __lint */
 204 
 205 #if defined(__amd64)
 206 
 207         ENTRY(on_trap)
 208         movw    %si, OT_PROT(%rdi)              /* ot_prot = prot */
 209         movw    $0, OT_TRAP(%rdi)               /* ot_trap = 0 */
 210         leaq    on_trap_trampoline(%rip), %rdx  /* rdx = &on_trap_trampoline */
 211         movq    %rdx, OT_TRAMPOLINE(%rdi)       /* ot_trampoline = rdx */
 212         xorl    %ecx, %ecx
 213         movq    %rcx, OT_HANDLE(%rdi)           /* ot_handle = NULL */
 214         movq    %rcx, OT_PAD1(%rdi)             /* ot_pad1 = NULL */
 215         movq    %gs:CPU_THREAD, %rdx            /* rdx = curthread */
 216         movq    T_ONTRAP(%rdx), %rcx            /* rcx = curthread->t_ontrap */
 217         cmpq    %rdi, %rcx                      /* if (otp == %rcx)     */
 218         je      0f                              /*      don't modify t_ontrap */
 219 
 220         movq    %rcx, OT_PREV(%rdi)             /* ot_prev = t_ontrap */
 221         movq    %rdi, T_ONTRAP(%rdx)            /* curthread->t_ontrap = otp */
 222 
 223 0:      addq    $OT_JMPBUF, %rdi                /* &ot_jmpbuf */
 224         jmp     setjmp
 225         SET_SIZE(on_trap)
 226 
 227 #elif defined(__i386)
 228 
 229         ENTRY(on_trap)
 230         movl    4(%esp), %eax                   /* %eax = otp */
 231         movl    8(%esp), %edx                   /* %edx = prot */
 232 
 233         movw    %dx, OT_PROT(%eax)              /* ot_prot = prot */
 234         movw    $0, OT_TRAP(%eax)               /* ot_trap = 0 */
 235         leal    on_trap_trampoline, %edx        /* %edx = &on_trap_trampoline */
 236         movl    %edx, OT_TRAMPOLINE(%eax)       /* ot_trampoline = %edx */
 237         movl    $0, OT_HANDLE(%eax)             /* ot_handle = NULL */
 238         movl    $0, OT_PAD1(%eax)               /* ot_pad1 = NULL */
 239         movl    %gs:CPU_THREAD, %edx            /* %edx = curthread */
 240         movl    T_ONTRAP(%edx), %ecx            /* %ecx = curthread->t_ontrap */
 241         cmpl    %eax, %ecx                      /* if (otp == %ecx) */
 242         je      0f                              /*    don't modify t_ontrap */
 243 
 244         movl    %ecx, OT_PREV(%eax)             /* ot_prev = t_ontrap */
 245         movl    %eax, T_ONTRAP(%edx)            /* curthread->t_ontrap = otp */
 246 
 247 0:      addl    $OT_JMPBUF, %eax                /* %eax = &ot_jmpbuf */
 248         movl    %eax, 4(%esp)                   /* put %eax back on the stack */
 249         jmp     setjmp                          /* let setjmp do the rest */
 250         SET_SIZE(on_trap)
 251 
 252 #endif  /* __i386 */
 253 #endif  /* __lint */
 254 
 255 /*
 256  * Setjmp and longjmp implement non-local gotos using state vectors
 257  * type label_t.
 258  */
 259 
 260 #if defined(__lint)
 261 
 262 /* ARGSUSED */
 263 int
 264 setjmp(label_t *lp)
 265 { return (0); }
 266 
 267 /* ARGSUSED */
 268 void
 269 longjmp(label_t *lp)
 270 {}
 271 
 272 #else   /* __lint */
 273 
 274 #if LABEL_PC != 0
 275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
 276 #endif  /* LABEL_PC != 0 */
 277 
 278 #if defined(__amd64)
 279 
 280         ENTRY(setjmp)
 281         movq    %rsp, LABEL_SP(%rdi)
 282         movq    %rbp, LABEL_RBP(%rdi)
 283         movq    %rbx, LABEL_RBX(%rdi)
 284         movq    %r12, LABEL_R12(%rdi)
 285         movq    %r13, LABEL_R13(%rdi)
 286         movq    %r14, LABEL_R14(%rdi)
 287         movq    %r15, LABEL_R15(%rdi)
 288         movq    (%rsp), %rdx            /* return address */
 289         movq    %rdx, (%rdi)            /* LABEL_PC is 0 */
 290         xorl    %eax, %eax              /* return 0 */
 291         ret
 292         SET_SIZE(setjmp)
 293 
 294         ENTRY(longjmp)
 295         movq    LABEL_SP(%rdi), %rsp
 296         movq    LABEL_RBP(%rdi), %rbp
 297         movq    LABEL_RBX(%rdi), %rbx
 298         movq    LABEL_R12(%rdi), %r12
 299         movq    LABEL_R13(%rdi), %r13
 300         movq    LABEL_R14(%rdi), %r14
 301         movq    LABEL_R15(%rdi), %r15
 302         movq    (%rdi), %rdx            /* return address; LABEL_PC is 0 */
 303         movq    %rdx, (%rsp)
 304         xorl    %eax, %eax
 305         incl    %eax                    /* return 1 */
 306         ret
 307         SET_SIZE(longjmp)
 308 
 309 #elif defined(__i386)
 310 
 311         ENTRY(setjmp)
 312         movl    4(%esp), %edx           /* address of save area */
 313         movl    %ebp, LABEL_EBP(%edx)
 314         movl    %ebx, LABEL_EBX(%edx)
 315         movl    %esi, LABEL_ESI(%edx)
 316         movl    %edi, LABEL_EDI(%edx)
 317         movl    %esp, 4(%edx)
 318         movl    (%esp), %ecx            /* %eip (return address) */
 319         movl    %ecx, (%edx)            /* LABEL_PC is 0 */
 320         subl    %eax, %eax              /* return 0 */
 321         ret
 322         SET_SIZE(setjmp)
 323 
 324         ENTRY(longjmp)
 325         movl    4(%esp), %edx           /* address of save area */
 326         movl    LABEL_EBP(%edx), %ebp
 327         movl    LABEL_EBX(%edx), %ebx
 328         movl    LABEL_ESI(%edx), %esi
 329         movl    LABEL_EDI(%edx), %edi
 330         movl    4(%edx), %esp
 331         movl    (%edx), %ecx            /* %eip (return addr); LABEL_PC is 0 */
 332         movl    $1, %eax
 333         addl    $4, %esp                /* pop ret adr */
 334         jmp     *%ecx                   /* indirect */
 335         SET_SIZE(longjmp)
 336 
 337 #endif  /* __i386 */
 338 #endif  /* __lint */
 339 
 340 /*
 341  * if a() calls b() calls caller(),
 342  * caller() returns return address in a().
 343  * (Note: We assume a() and b() are C routines which do the normal entry/exit
 344  *  sequence.)
 345  */
 346 
 347 #if defined(__lint)
 348 
 349 caddr_t
 350 caller(void)
 351 { return (0); }
 352 
 353 #else   /* __lint */
 354 
 355 #if defined(__amd64)
 356 
 357         ENTRY(caller)
 358         movq    8(%rbp), %rax           /* b()'s return pc, in a() */
 359         ret
 360         SET_SIZE(caller)
 361 
 362 #elif defined(__i386)
 363 
 364         ENTRY(caller)
 365         movl    4(%ebp), %eax           /* b()'s return pc, in a() */
 366         ret
 367         SET_SIZE(caller)
 368 
 369 #endif  /* __i386 */
 370 #endif  /* __lint */
 371 
 372 /*
 373  * if a() calls callee(), callee() returns the
 374  * return address in a();
 375  */
 376 
 377 #if defined(__lint)
 378 
 379 caddr_t
 380 callee(void)
 381 { return (0); }
 382 
 383 #else   /* __lint */
 384 
 385 #if defined(__amd64)
 386 
 387         ENTRY(callee)
 388         movq    (%rsp), %rax            /* callee()'s return pc, in a() */
 389         ret
 390         SET_SIZE(callee)
 391 
 392 #elif defined(__i386)
 393 
 394         ENTRY(callee)
 395         movl    (%esp), %eax            /* callee()'s return pc, in a() */
 396         ret
 397         SET_SIZE(callee)
 398 
 399 #endif  /* __i386 */
 400 #endif  /* __lint */
 401 
 402 /*
 403  * return the current frame pointer
 404  */
 405 
 406 #if defined(__lint)
 407 
 408 greg_t
 409 getfp(void)
 410 { return (0); }
 411 
 412 #else   /* __lint */
 413 
 414 #if defined(__amd64)
 415 
 416         ENTRY(getfp)
 417         movq    %rbp, %rax
 418         ret
 419         SET_SIZE(getfp)
 420 
 421 #elif defined(__i386)
 422 
 423         ENTRY(getfp)
 424         movl    %ebp, %eax
 425         ret
 426         SET_SIZE(getfp)
 427 
 428 #endif  /* __i386 */
 429 #endif  /* __lint */
 430 
 431 /*
 432  * Invalidate a single page table entry in the TLB
 433  */
 434 
 435 #if defined(__lint)
 436 
 437 /* ARGSUSED */
 438 void
 439 mmu_invlpg(caddr_t m)
 440 {}
 441 
 442 #else   /* __lint */
 443 
 444         ENTRY(mmu_invlpg)


 445         invlpg  (%rdi)
 446         ret
 447         SET_SIZE(mmu_invlpg)
 448 









 449 #endif  /* __lint */
 450 
 451 
 452 /*
 453  * Get/Set the value of various control registers
 454  */
 455 
 456 #if defined(__lint)
 457 
 458 ulong_t
 459 getcr0(void)
 460 { return (0); }
 461 
 462 /* ARGSUSED */
 463 void
 464 setcr0(ulong_t value)
 465 {}
 466 
 467 ulong_t
 468 getcr2(void)
 469 { return (0); }
 470 
 471 ulong_t
 472 getcr3(void)
 473 { return (0); }
 474 
 475 #if !defined(__xpv)
 476 /* ARGSUSED */
 477 void
 478 setcr3(ulong_t val)
 479 {}
 480 
 481 void
 482 reload_cr3(void)
 483 {}
 484 #endif
 485 
 486 ulong_t
 487 getcr4(void)
 488 { return (0); }
 489 
 490 /* ARGSUSED */
 491 void
 492 setcr4(ulong_t val)
 493 {}
 494 
 495 #if defined(__amd64)
 496 
 497 ulong_t
 498 getcr8(void)
 499 { return (0); }
 500 
 501 /* ARGSUSED */
 502 void
 503 setcr8(ulong_t val)
 504 {}
 505 
 506 #endif  /* __amd64 */
 507 
 508 #else   /* __lint */
 509 
 510 #if defined(__amd64)
 511 
 512         ENTRY(getcr0)
 513         movq    %cr0, %rax
 514         ret
 515         SET_SIZE(getcr0)
 516 
 517         ENTRY(setcr0)
 518         movq    %rdi, %cr0
 519         ret
 520         SET_SIZE(setcr0)
 521 
 522         ENTRY(getcr2)
 523 #if defined(__xpv)
 524         movq    %gs:CPU_VCPU_INFO, %rax
 525         movq    VCPU_INFO_ARCH_CR2(%rax), %rax
 526 #else
 527         movq    %cr2, %rax
 528 #endif
 529         ret
 530         SET_SIZE(getcr2)
 531 
 532         ENTRY(getcr3)
 533         movq    %cr3, %rax
 534         ret
 535         SET_SIZE(getcr3)
 536 
 537 #if !defined(__xpv)
 538 
 539         ENTRY(setcr3)
 540         movq    %rdi, %cr3
 541         ret
 542         SET_SIZE(setcr3)
 543 
 544         ENTRY(reload_cr3)
 545         movq    %cr3, %rdi
 546         movq    %rdi, %cr3
 547         ret
 548         SET_SIZE(reload_cr3)
 549 
 550 #endif  /* __xpv */
 551 
 552         ENTRY(getcr4)
 553         movq    %cr4, %rax
 554         ret
 555         SET_SIZE(getcr4)
 556 
 557         ENTRY(setcr4)
 558         movq    %rdi, %cr4
 559         ret
 560         SET_SIZE(setcr4)
 561 
 562         ENTRY(getcr8)
 563         movq    %cr8, %rax
 564         ret
 565         SET_SIZE(getcr8)
 566 
 567         ENTRY(setcr8)
 568         movq    %rdi, %cr8
 569         ret
 570         SET_SIZE(setcr8)
 571 
 572 #elif defined(__i386)
 573 
 574         ENTRY(getcr0)
 575         movl    %cr0, %eax
 576         ret
 577         SET_SIZE(getcr0)
 578 
 579         ENTRY(setcr0)
 580         movl    4(%esp), %eax
 581         movl    %eax, %cr0
 582         ret
 583         SET_SIZE(setcr0)
 584 
 585         /*
 586          * "lock mov %cr0" is used on processors which indicate it is
 587          * supported via CPUID. Normally the 32 bit TPR is accessed via
 588          * the local APIC.
 589          */
 590         ENTRY(getcr8)
 591         lock
 592         movl    %cr0, %eax
 593         ret
 594         SET_SIZE(getcr8)
 595 
 596         ENTRY(setcr8)
 597         movl    4(%esp), %eax
 598         lock
 599         movl    %eax, %cr0
 600         ret
 601         SET_SIZE(setcr8)
 602 
 603         ENTRY(getcr2)
 604 #if defined(__xpv)
 605         movl    %gs:CPU_VCPU_INFO, %eax
 606         movl    VCPU_INFO_ARCH_CR2(%eax), %eax
 607 #else
 608         movl    %cr2, %eax
 609 #endif
 610         ret
 611         SET_SIZE(getcr2)
 612 
 613         ENTRY(getcr3)
 614         movl    %cr3, %eax
 615         ret
 616         SET_SIZE(getcr3)
 617 
 618 #if !defined(__xpv)
 619 
 620         ENTRY(setcr3)
 621         movl    4(%esp), %eax
 622         movl    %eax, %cr3
 623         ret
 624         SET_SIZE(setcr3)
 625 
 626         ENTRY(reload_cr3)
 627         movl    %cr3, %eax
 628         movl    %eax, %cr3
 629         ret
 630         SET_SIZE(reload_cr3)
 631 
 632 #endif  /* __xpv */
 633 
 634         ENTRY(getcr4)
 635         movl    %cr4, %eax
 636         ret
 637         SET_SIZE(getcr4)
 638 
 639         ENTRY(setcr4)
 640         movl    4(%esp), %eax
 641         movl    %eax, %cr4
 642         ret
 643         SET_SIZE(setcr4)
 644 
 645 #endif  /* __i386 */
 646 #endif  /* __lint */
 647 
 648 #if defined(__lint)
 649 
 650 /*ARGSUSED*/
 651 uint32_t
 652 __cpuid_insn(struct cpuid_regs *regs)
 653 { return (0); }
 654 
 655 #else   /* __lint */
 656 
 657 #if defined(__amd64)
 658 
 659         ENTRY(__cpuid_insn)
 660         movq    %rbx, %r8
 661         movq    %rcx, %r9
 662         movq    %rdx, %r11
 663         movl    (%rdi), %eax            /* %eax = regs->cp_eax */
 664         movl    0x4(%rdi), %ebx         /* %ebx = regs->cp_ebx */
 665         movl    0x8(%rdi), %ecx         /* %ecx = regs->cp_ecx */
 666         movl    0xc(%rdi), %edx         /* %edx = regs->cp_edx */
 667         cpuid
 668         movl    %eax, (%rdi)            /* regs->cp_eax = %eax */
 669         movl    %ebx, 0x4(%rdi)         /* regs->cp_ebx = %ebx */
 670         movl    %ecx, 0x8(%rdi)         /* regs->cp_ecx = %ecx */
 671         movl    %edx, 0xc(%rdi)         /* regs->cp_edx = %edx */
 672         movq    %r8, %rbx
 673         movq    %r9, %rcx
 674         movq    %r11, %rdx
 675         ret
 676         SET_SIZE(__cpuid_insn)
 677 
 678 #elif defined(__i386)
 679 
 680         ENTRY(__cpuid_insn)
 681         pushl   %ebp
 682         movl    0x8(%esp), %ebp         /* %ebp = regs */
 683         pushl   %ebx
 684         pushl   %ecx
 685         pushl   %edx
 686         movl    (%ebp), %eax            /* %eax = regs->cp_eax */
 687         movl    0x4(%ebp), %ebx         /* %ebx = regs->cp_ebx */
 688         movl    0x8(%ebp), %ecx         /* %ecx = regs->cp_ecx */
 689         movl    0xc(%ebp), %edx         /* %edx = regs->cp_edx */
 690         cpuid
 691         movl    %eax, (%ebp)            /* regs->cp_eax = %eax */
 692         movl    %ebx, 0x4(%ebp)         /* regs->cp_ebx = %ebx */
 693         movl    %ecx, 0x8(%ebp)         /* regs->cp_ecx = %ecx */
 694         movl    %edx, 0xc(%ebp)         /* regs->cp_edx = %edx */
 695         popl    %edx
 696         popl    %ecx
 697         popl    %ebx
 698         popl    %ebp
 699         ret
 700         SET_SIZE(__cpuid_insn)
 701 
 702 #endif  /* __i386 */
 703 #endif  /* __lint */
 704 
 705 #if defined(__lint)
 706 
 707 /*ARGSUSED*/
 708 void
 709 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
 710 {}
 711 
 712 #else   /* __lint */
 713 
 714 #if defined(__amd64)
 715 
 716         ENTRY_NP(i86_monitor)
 717         pushq   %rbp
 718         movq    %rsp, %rbp
 719         movq    %rdi, %rax              /* addr */
 720         movq    %rsi, %rcx              /* extensions */
 721         /* rdx contains input arg3: hints */
 722         clflush (%rax)
 723         .byte   0x0f, 0x01, 0xc8        /* monitor */
 724         leave
 725         ret
 726         SET_SIZE(i86_monitor)
 727 
 728 #elif defined(__i386)
 729 
 730 ENTRY_NP(i86_monitor)
 731         pushl   %ebp
 732         movl    %esp, %ebp
 733         movl    0x8(%ebp),%eax          /* addr */
 734         movl    0xc(%ebp),%ecx          /* extensions */
 735         movl    0x10(%ebp),%edx         /* hints */
 736         clflush (%eax)
 737         .byte   0x0f, 0x01, 0xc8        /* monitor */
 738         leave
 739         ret
 740         SET_SIZE(i86_monitor)
 741 
 742 #endif  /* __i386 */
 743 #endif  /* __lint */
 744 
 745 #if defined(__lint)
 746 
 747 /*ARGSUSED*/
 748 void
 749 i86_mwait(uint32_t data, uint32_t extensions)
 750 {}
 751 
 752 #else   /* __lint */
 753 
 754 #if defined(__amd64)
 755 
 756         ENTRY_NP(i86_mwait)
 757         pushq   %rbp
 758         movq    %rsp, %rbp
 759         movq    %rdi, %rax              /* data */
 760         movq    %rsi, %rcx              /* extensions */
 761         .byte   0x0f, 0x01, 0xc9        /* mwait */
 762         leave
 763         ret
 764         SET_SIZE(i86_mwait)
 765 
 766 #elif defined(__i386)
 767 
 768         ENTRY_NP(i86_mwait)
 769         pushl   %ebp
 770         movl    %esp, %ebp
 771         movl    0x8(%ebp),%eax          /* data */
 772         movl    0xc(%ebp),%ecx          /* extensions */
 773         .byte   0x0f, 0x01, 0xc9        /* mwait */
 774         leave
 775         ret
 776         SET_SIZE(i86_mwait)
 777 
 778 #endif  /* __i386 */
 779 #endif  /* __lint */
 780 
 781 #if defined(__xpv)
 782         /*
 783          * Defined in C
 784          */
 785 #else
 786 
 787 #if defined(__lint)
 788 
 789 hrtime_t
 790 tsc_read(void)
 791 {
 792         return (0);
 793 }
 794 
 795 #else   /* __lint */
 796 
 797 #if defined(__amd64)
 798 
 799         ENTRY_NP(tsc_read)
 800         movq    %rbx, %r11
 801         movl    $0, %eax
 802         cpuid
 803         rdtsc
 804         movq    %r11, %rbx
 805         shlq    $32, %rdx
 806         orq     %rdx, %rax
 807         ret
 808         .globl _tsc_mfence_start
 809 _tsc_mfence_start:
 810         mfence
 811         rdtsc
 812         shlq    $32, %rdx
 813         orq     %rdx, %rax
 814         ret
 815         .globl _tsc_mfence_end
 816 _tsc_mfence_end:
 817         .globl _tscp_start
 818 _tscp_start:
 819         .byte   0x0f, 0x01, 0xf9        /* rdtscp instruction */
 820         shlq    $32, %rdx
 821         orq     %rdx, %rax
 822         ret
 823         .globl _tscp_end
 824 _tscp_end:
 825         .globl _no_rdtsc_start
 826 _no_rdtsc_start:
 827         xorl    %edx, %edx
 828         xorl    %eax, %eax
 829         ret
 830         .globl _no_rdtsc_end
 831 _no_rdtsc_end:
 832         .globl _tsc_lfence_start
 833 _tsc_lfence_start:
 834         lfence
 835         rdtsc
 836         shlq    $32, %rdx
 837         orq     %rdx, %rax
 838         ret
 839         .globl _tsc_lfence_end
 840 _tsc_lfence_end:
 841         SET_SIZE(tsc_read)
 842 
 843 #else /* __i386 */
 844 
 845         ENTRY_NP(tsc_read)
 846         pushl   %ebx
 847         movl    $0, %eax
 848         cpuid
 849         rdtsc
 850         popl    %ebx
 851         ret
 852         .globl _tsc_mfence_start
 853 _tsc_mfence_start:
 854         mfence
 855         rdtsc
 856         ret
 857         .globl _tsc_mfence_end
 858 _tsc_mfence_end:
 859         .globl  _tscp_start
 860 _tscp_start:
 861         .byte   0x0f, 0x01, 0xf9        /* rdtscp instruction */
 862         ret
 863         .globl _tscp_end
 864 _tscp_end:
 865         .globl _no_rdtsc_start
 866 _no_rdtsc_start:
 867         xorl    %edx, %edx
 868         xorl    %eax, %eax
 869         ret
 870         .globl _no_rdtsc_end
 871 _no_rdtsc_end:
 872         .globl _tsc_lfence_start
 873 _tsc_lfence_start:
 874         lfence
 875         rdtsc
 876         ret
 877         .globl _tsc_lfence_end
 878 _tsc_lfence_end:
 879         SET_SIZE(tsc_read)
 880 
 881 #endif  /* __i386 */
 882 
 883 #endif  /* __lint */
 884 
 885 
 886 #endif  /* __xpv */
 887 
 888 #ifdef __lint
 889 /*
 890  * Do not use this function for obtaining clock tick.  This
 891  * is called by callers who do not need to have a guarenteed
 892  * correct tick value.  The proper routine to use is tsc_read().
 893  */
 894 u_longlong_t
 895 randtick(void)
 896 {
 897         return (0);
 898 }
 899 #else
 900 #if defined(__amd64)
 901         ENTRY_NP(randtick)
 902         rdtsc
 903         shlq    $32, %rdx
 904         orq     %rdx, %rax
 905         ret
 906         SET_SIZE(randtick)
 907 #else
 908         ENTRY_NP(randtick)
 909         rdtsc
 910         ret
 911         SET_SIZE(randtick)
 912 #endif /* __i386 */
 913 #endif /* __lint */
 914 /*
 915  * Insert entryp after predp in a doubly linked list.
 916  */
 917 
 918 #if defined(__lint)
 919 
 920 /*ARGSUSED*/
 921 void
 922 _insque(caddr_t entryp, caddr_t predp)
 923 {}
 924 
 925 #else   /* __lint */
 926 
 927 #if defined(__amd64)
 928 
 929         ENTRY(_insque)
 930         movq    (%rsi), %rax            /* predp->forw                       */
 931         movq    %rsi, CPTRSIZE(%rdi)    /* entryp->back = predp              */
 932         movq    %rax, (%rdi)            /* entryp->forw = predp->forw     */
 933         movq    %rdi, (%rsi)            /* predp->forw = entryp              */
 934         movq    %rdi, CPTRSIZE(%rax)    /* predp->forw->back = entryp     */
 935         ret
 936         SET_SIZE(_insque)
 937 
 938 #elif defined(__i386)
 939 
 940         ENTRY(_insque)
 941         movl    8(%esp), %edx
 942         movl    4(%esp), %ecx
 943         movl    (%edx), %eax            /* predp->forw                       */
 944         movl    %edx, CPTRSIZE(%ecx)    /* entryp->back = predp              */
 945         movl    %eax, (%ecx)            /* entryp->forw = predp->forw     */
 946         movl    %ecx, (%edx)            /* predp->forw = entryp              */
 947         movl    %ecx, CPTRSIZE(%eax)    /* predp->forw->back = entryp     */
 948         ret
 949         SET_SIZE(_insque)
 950 
 951 #endif  /* __i386 */
 952 #endif  /* __lint */
 953 
 954 /*
 955  * Remove entryp from a doubly linked list
 956  */
 957 
 958 #if defined(__lint)
 959 
 960 /*ARGSUSED*/
 961 void
 962 _remque(caddr_t entryp)
 963 {}
 964 
 965 #else   /* __lint */
 966 
 967 #if defined(__amd64)
 968 
 969         ENTRY(_remque)
 970         movq    (%rdi), %rax            /* entry->forw */
 971         movq    CPTRSIZE(%rdi), %rdx    /* entry->back */
 972         movq    %rax, (%rdx)            /* entry->back->forw = entry->forw */
 973         movq    %rdx, CPTRSIZE(%rax)    /* entry->forw->back = entry->back */
 974         ret
 975         SET_SIZE(_remque)
 976 
 977 #elif defined(__i386)
 978 
 979         ENTRY(_remque)
 980         movl    4(%esp), %ecx
 981         movl    (%ecx), %eax            /* entry->forw */
 982         movl    CPTRSIZE(%ecx), %edx    /* entry->back */
 983         movl    %eax, (%edx)            /* entry->back->forw = entry->forw */
 984         movl    %edx, CPTRSIZE(%eax)    /* entry->forw->back = entry->back */
 985         ret
 986         SET_SIZE(_remque)
 987 
 988 #endif  /* __i386 */
 989 #endif  /* __lint */
 990 
 991 /*
 992  * Returns the number of
 993  * non-NULL bytes in string argument.
 994  */
 995 
 996 #if defined(__lint)
 997 
 998 /* ARGSUSED */
 999 size_t
1000 strlen(const char *str)
1001 { return (0); }
1002 
1003 #else   /* __lint */
1004 
1005 #if defined(__amd64)
1006 
1007 /*
1008  * This is close to a simple transliteration of a C version of this
1009  * routine.  We should either just -make- this be a C version, or
1010  * justify having it in assembler by making it significantly faster.
1011  *
1012  * size_t
1013  * strlen(const char *s)
1014  * {
1015  *      const char *s0;
1016  * #if defined(DEBUG)
1017  *      if ((uintptr_t)s < KERNELBASE)
1018  *              panic(.str_panic_msg);
1019  * #endif
1020  *      for (s0 = s; *s; s++)
1021  *              ;
1022  *      return (s - s0);
1023  * }
1024  */
1025 
1026         ENTRY(strlen)
1027 #ifdef DEBUG
1028         movq    postbootkernelbase(%rip), %rax
1029         cmpq    %rax, %rdi
1030         jae     str_valid
1031         pushq   %rbp
1032         movq    %rsp, %rbp
1033         leaq    .str_panic_msg(%rip), %rdi
1034         xorl    %eax, %eax
1035         call    panic
1036 #endif  /* DEBUG */
1037 str_valid:
1038         cmpb    $0, (%rdi)
1039         movq    %rdi, %rax
1040         je      .null_found
1041         .align  4
1042 .strlen_loop:
1043         incq    %rdi
1044         cmpb    $0, (%rdi)
1045         jne     .strlen_loop
1046 .null_found:
1047         subq    %rax, %rdi
1048         movq    %rdi, %rax
1049         ret
1050         SET_SIZE(strlen)
1051 
1052 #elif defined(__i386)
1053 
1054         ENTRY(strlen)
1055 #ifdef DEBUG
1056         movl    postbootkernelbase, %eax
1057         cmpl    %eax, 4(%esp)
1058         jae     str_valid
1059         pushl   %ebp
1060         movl    %esp, %ebp
1061         pushl   $.str_panic_msg
1062         call    panic
1063 #endif /* DEBUG */
1064 
1065 str_valid:
1066         movl    4(%esp), %eax           /* %eax = string address */
1067         testl   $3, %eax                /* if %eax not word aligned */
1068         jnz     .not_word_aligned       /* goto .not_word_aligned */
1069         .align  4
1070 .word_aligned:
1071         movl    (%eax), %edx            /* move 1 word from (%eax) to %edx */
1072         movl    $0x7f7f7f7f, %ecx
1073         andl    %edx, %ecx              /* %ecx = %edx & 0x7f7f7f7f */
1074         addl    $4, %eax                /* next word */
1075         addl    $0x7f7f7f7f, %ecx       /* %ecx += 0x7f7f7f7f */
1076         orl     %edx, %ecx              /* %ecx |= %edx */
1077         andl    $0x80808080, %ecx       /* %ecx &= 0x80808080 */
1078         cmpl    $0x80808080, %ecx       /* if no null byte in this word */
1079         je      .word_aligned           /* goto .word_aligned */
1080         subl    $4, %eax                /* post-incremented */
1081 .not_word_aligned:
1082         cmpb    $0, (%eax)              /* if a byte in (%eax) is null */
1083         je      .null_found             /* goto .null_found */
1084         incl    %eax                    /* next byte */
1085         testl   $3, %eax                /* if %eax not word aligned */
1086         jnz     .not_word_aligned       /* goto .not_word_aligned */
1087         jmp     .word_aligned           /* goto .word_aligned */
1088         .align  4
1089 .null_found:
1090         subl    4(%esp), %eax           /* %eax -= string address */
1091         ret
1092         SET_SIZE(strlen)
1093 
1094 #endif  /* __i386 */
1095 
1096 #ifdef DEBUG
1097         .text
1098 .str_panic_msg:
1099         .string "strlen: argument below kernelbase"
1100 #endif /* DEBUG */
1101 
1102 #endif  /* __lint */
1103 
1104         /*
1105          * Berkeley 4.3 introduced symbolically named interrupt levels
1106          * as a way deal with priority in a machine independent fashion.
1107          * Numbered priorities are machine specific, and should be
1108          * discouraged where possible.
1109          *
1110          * Note, for the machine specific priorities there are
1111          * examples listed for devices that use a particular priority.
1112          * It should not be construed that all devices of that
1113          * type should be at that priority.  It is currently were
1114          * the current devices fit into the priority scheme based
1115          * upon time criticalness.
1116          *
1117          * The underlying assumption of these assignments is that
1118          * IPL 10 is the highest level from which a device
1119          * routine can call wakeup.  Devices that interrupt from higher
1120          * levels are restricted in what they can do.  If they need
1121          * kernels services they should schedule a routine at a lower
1122          * level (via software interrupt) to do the required
1123          * processing.
1124          *
1125          * Examples of this higher usage:
1126          *      Level   Usage
1127          *      14      Profiling clock (and PROM uart polling clock)
1128          *      12      Serial ports
1129          *
1130          * The serial ports request lower level processing on level 6.
1131          *
1132          * Also, almost all splN routines (where N is a number or a
1133          * mnemonic) will do a RAISE(), on the assumption that they are
1134          * never used to lower our priority.
1135          * The exceptions are:
1136          *      spl8()          Because you can't be above 15 to begin with!
1137          *      splzs()         Because this is used at boot time to lower our
1138          *                      priority, to allow the PROM to poll the uart.
1139          *      spl0()          Used to lower priority to 0.
1140          */
1141 
1142 #if defined(__lint)
1143 
1144 int spl0(void)          { return (0); }
1145 int spl6(void)          { return (0); }
1146 int spl7(void)          { return (0); }
1147 int spl8(void)          { return (0); }
1148 int splhigh(void)       { return (0); }
1149 int splhi(void)         { return (0); }
1150 int splzs(void)         { return (0); }
1151 
1152 /* ARGSUSED */
1153 void
1154 splx(int level)
1155 {}
1156 
1157 #else   /* __lint */
1158 
1159 #if defined(__amd64)
1160 
1161 #define SETPRI(level) \
1162         movl    $/**/level, %edi;       /* new priority */              \
1163         jmp     do_splx                 /* redirect to do_splx */
1164 
1165 #define RAISE(level) \
1166         movl    $/**/level, %edi;       /* new priority */              \
1167         jmp     splr                    /* redirect to splr */
1168 
1169 #elif defined(__i386)
1170 
1171 #define SETPRI(level) \
1172         pushl   $/**/level;     /* new priority */                      \
1173         call    do_splx;        /* invoke common splx code */           \
1174         addl    $4, %esp;       /* unstack arg */                       \
1175         ret
1176 
1177 #define RAISE(level) \
1178         pushl   $/**/level;     /* new priority */                      \
1179         call    splr;           /* invoke common splr code */           \
1180         addl    $4, %esp;       /* unstack args */                      \
1181         ret
1182 
1183 #endif  /* __i386 */
1184 
1185         /* locks out all interrupts, including memory errors */
1186         ENTRY(spl8)
1187         SETPRI(15)
1188         SET_SIZE(spl8)
1189 
1190         /* just below the level that profiling runs */
1191         ENTRY(spl7)
1192         RAISE(13)
1193         SET_SIZE(spl7)
1194 
1195         /* sun specific - highest priority onboard serial i/o asy ports */
1196         ENTRY(splzs)
1197         SETPRI(12)      /* Can't be a RAISE, as it's used to lower us */
1198         SET_SIZE(splzs)
1199 
1200         ENTRY(splhi)
1201         ALTENTRY(splhigh)
1202         ALTENTRY(spl6)
1203         ALTENTRY(i_ddi_splhigh)
1204 
1205         RAISE(DISP_LEVEL)
1206 
1207         SET_SIZE(i_ddi_splhigh)
1208         SET_SIZE(spl6)
1209         SET_SIZE(splhigh)
1210         SET_SIZE(splhi)
1211 
1212         /* allow all interrupts */
1213         ENTRY(spl0)
1214         SETPRI(0)
1215         SET_SIZE(spl0)
1216 
1217 
1218         /* splx implementation */
1219         ENTRY(splx)
1220         jmp     do_splx         /* redirect to common splx code */
1221         SET_SIZE(splx)
1222 
1223 #endif  /* __lint */
1224 
1225 #if defined(__i386)
1226 
1227 /*
1228  * Read and write the %gs register
1229  */
1230 
1231 #if defined(__lint)
1232 
1233 /*ARGSUSED*/
1234 uint16_t
1235 getgs(void)
1236 { return (0); }
1237 
1238 /*ARGSUSED*/
1239 void
1240 setgs(uint16_t sel)
1241 {}
1242 
1243 #else   /* __lint */
1244 
1245         ENTRY(getgs)
1246         clr     %eax
1247         movw    %gs, %ax
1248         ret
1249         SET_SIZE(getgs)
1250 
1251         ENTRY(setgs)
1252         movw    4(%esp), %gs
1253         ret
1254         SET_SIZE(setgs)
1255 
1256 #endif  /* __lint */
1257 #endif  /* __i386 */
1258 
1259 #if defined(__lint)
1260 
1261 void
1262 pc_reset(void)
1263 {}
1264 
1265 void
1266 efi_reset(void)
1267 {}
1268 
1269 #else   /* __lint */
1270 
1271         ENTRY(wait_500ms)
1272 #if defined(__amd64)
1273         pushq   %rbx
1274 #elif defined(__i386)
1275         push    %ebx
1276 #endif
1277         movl    $50000, %ebx
1278 1:
1279         call    tenmicrosec
1280         decl    %ebx
1281         jnz     1b
1282 #if defined(__amd64)
1283         popq    %rbx
1284 #elif defined(__i386)
1285         pop     %ebx
1286 #endif
1287         ret     
1288         SET_SIZE(wait_500ms)
1289 
1290 #define RESET_METHOD_KBC        1
1291 #define RESET_METHOD_PORT92     2
1292 #define RESET_METHOD_PCI        4
1293 
1294         DGDEF3(pc_reset_methods, 4, 8)
1295         .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1296 
1297         ENTRY(pc_reset)
1298 
1299 #if defined(__i386)
1300         testl   $RESET_METHOD_KBC, pc_reset_methods
1301 #elif defined(__amd64)
1302         testl   $RESET_METHOD_KBC, pc_reset_methods(%rip)
1303 #endif
1304         jz      1f
1305 
1306         /
1307         / Try the classic keyboard controller-triggered reset.
1308         /
1309         movw    $0x64, %dx
1310         movb    $0xfe, %al
1311         outb    (%dx)
1312 
1313         / Wait up to 500 milliseconds here for the keyboard controller
1314         / to pull the reset line.  On some systems where the keyboard
1315         / controller is slow to pull the reset line, the next reset method
1316         / may be executed (which may be bad if those systems hang when the
1317         / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1318         / and Ferrari 4000 (doesn't like the cf9 reset method))
1319 
1320         call    wait_500ms
1321 
1322 1:
1323 #if defined(__i386)
1324         testl   $RESET_METHOD_PORT92, pc_reset_methods
1325 #elif defined(__amd64)
1326         testl   $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1327 #endif
1328         jz      3f
1329 
1330         /
1331         / Try port 0x92 fast reset
1332         /
1333         movw    $0x92, %dx
1334         inb     (%dx)
1335         cmpb    $0xff, %al      / If port's not there, we should get back 0xFF
1336         je      1f
1337         testb   $1, %al         / If bit 0
1338         jz      2f              / is clear, jump to perform the reset
1339         andb    $0xfe, %al      / otherwise,
1340         outb    (%dx)           / clear bit 0 first, then
1341 2:
1342         orb     $1, %al         / Set bit 0
1343         outb    (%dx)           / and reset the system
1344 1:
1345 
1346         call    wait_500ms
1347 
1348 3:
1349 #if defined(__i386)
1350         testl   $RESET_METHOD_PCI, pc_reset_methods
1351 #elif defined(__amd64)
1352         testl   $RESET_METHOD_PCI, pc_reset_methods(%rip)
1353 #endif
1354         jz      4f
1355 
1356         / Try the PCI (soft) reset vector (should work on all modern systems,
1357         / but has been shown to cause problems on 450NX systems, and some newer
1358         / systems (e.g. ATI IXP400-equipped systems))
1359         / When resetting via this method, 2 writes are required.  The first
1360         / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1361         / power cycle).
1362         / The reset occurs on the second write, during bit 2's transition from
1363         / 0->1.
1364         movw    $0xcf9, %dx
1365         movb    $0x2, %al       / Reset mode = hard, no power cycle
1366         outb    (%dx)
1367         movb    $0x6, %al
1368         outb    (%dx)
1369 
1370         call    wait_500ms
1371 
1372 4:
1373         /
1374         / port 0xcf9 failed also.  Last-ditch effort is to
1375         / triple-fault the CPU.
1376         / Also, use triple fault for EFI firmware
1377         /
1378         ENTRY(efi_reset)
1379 #if defined(__amd64)
1380         pushq   $0x0
1381         pushq   $0x0            / IDT base of 0, limit of 0 + 2 unused bytes
1382         lidt    (%rsp)
1383 #elif defined(__i386)
1384         pushl   $0x0
1385         pushl   $0x0            / IDT base of 0, limit of 0 + 2 unused bytes
1386         lidt    (%esp)
1387 #endif
1388         int     $0x0            / Trigger interrupt, generate triple-fault
1389 
1390         cli
1391         hlt                     / Wait forever
1392         /*NOTREACHED*/
1393         SET_SIZE(efi_reset)
1394         SET_SIZE(pc_reset)
1395 
1396 #endif  /* __lint */
1397 
1398 /*
1399  * C callable in and out routines
1400  */
1401 
1402 #if defined(__lint)
1403 
1404 /* ARGSUSED */
1405 void
1406 outl(int port_address, uint32_t val)
1407 {}
1408 
1409 #else   /* __lint */
1410 
1411 #if defined(__amd64)
1412 
1413         ENTRY(outl)
1414         movw    %di, %dx
1415         movl    %esi, %eax
1416         outl    (%dx)
1417         ret
1418         SET_SIZE(outl)
1419 
1420 #elif defined(__i386)
1421 
1422         .set    PORT, 4
1423         .set    VAL, 8
1424 
1425         ENTRY(outl)
1426         movw    PORT(%esp), %dx
1427         movl    VAL(%esp), %eax
1428         outl    (%dx)
1429         ret
1430         SET_SIZE(outl)
1431 
1432 #endif  /* __i386 */
1433 #endif  /* __lint */
1434 
1435 #if defined(__lint)
1436 
1437 /* ARGSUSED */
1438 void
1439 outw(int port_address, uint16_t val)
1440 {}
1441 
1442 #else   /* __lint */
1443 
1444 #if defined(__amd64)
1445 
1446         ENTRY(outw)
1447         movw    %di, %dx
1448         movw    %si, %ax
1449         D16 outl (%dx)          /* XX64 why not outw? */
1450         ret
1451         SET_SIZE(outw)
1452 
1453 #elif defined(__i386)
1454 
1455         ENTRY(outw)
1456         movw    PORT(%esp), %dx
1457         movw    VAL(%esp), %ax
1458         D16 outl (%dx)
1459         ret
1460         SET_SIZE(outw)
1461 
1462 #endif  /* __i386 */
1463 #endif  /* __lint */
1464 
1465 #if defined(__lint)
1466 
1467 /* ARGSUSED */
1468 void
1469 outb(int port_address, uint8_t val)
1470 {}
1471 
1472 #else   /* __lint */
1473 
1474 #if defined(__amd64)
1475 
1476         ENTRY(outb)
1477         movw    %di, %dx
1478         movb    %sil, %al
1479         outb    (%dx)
1480         ret
1481         SET_SIZE(outb)
1482 
1483 #elif defined(__i386)
1484 
1485         ENTRY(outb)
1486         movw    PORT(%esp), %dx
1487         movb    VAL(%esp), %al
1488         outb    (%dx)
1489         ret
1490         SET_SIZE(outb)
1491 
1492 #endif  /* __i386 */
1493 #endif  /* __lint */
1494 
1495 #if defined(__lint)
1496 
1497 /* ARGSUSED */
1498 uint32_t
1499 inl(int port_address)
1500 { return (0); }
1501 
1502 #else   /* __lint */
1503 
1504 #if defined(__amd64)
1505 
1506         ENTRY(inl)
1507         xorl    %eax, %eax
1508         movw    %di, %dx
1509         inl     (%dx)
1510         ret
1511         SET_SIZE(inl)
1512 
1513 #elif defined(__i386)
1514 
1515         ENTRY(inl)
1516         movw    PORT(%esp), %dx
1517         inl     (%dx)
1518         ret
1519         SET_SIZE(inl)
1520 
1521 #endif  /* __i386 */
1522 #endif  /* __lint */
1523 
1524 #if defined(__lint)
1525 
1526 /* ARGSUSED */
1527 uint16_t
1528 inw(int port_address)
1529 { return (0); }
1530 
1531 #else   /* __lint */
1532 
1533 #if defined(__amd64)
1534 
1535         ENTRY(inw)
1536         xorl    %eax, %eax
1537         movw    %di, %dx
1538         D16 inl (%dx)
1539         ret
1540         SET_SIZE(inw)
1541 
1542 #elif defined(__i386)
1543 
1544         ENTRY(inw)
1545         subl    %eax, %eax
1546         movw    PORT(%esp), %dx
1547         D16 inl (%dx)
1548         ret
1549         SET_SIZE(inw)
1550 
1551 #endif  /* __i386 */
1552 #endif  /* __lint */
1553 
1554 
1555 #if defined(__lint)
1556 
1557 /* ARGSUSED */
1558 uint8_t
1559 inb(int port_address)
1560 { return (0); }
1561 
1562 #else   /* __lint */
1563 
1564 #if defined(__amd64)
1565 
1566         ENTRY(inb)
1567         xorl    %eax, %eax
1568         movw    %di, %dx
1569         inb     (%dx)
1570         ret
1571         SET_SIZE(inb)
1572 
1573 #elif defined(__i386)
1574 
1575         ENTRY(inb)
1576         subl    %eax, %eax
1577         movw    PORT(%esp), %dx
1578         inb     (%dx)
1579         ret
1580         SET_SIZE(inb)
1581 
1582 #endif  /* __i386 */
1583 #endif  /* __lint */
1584 
1585 
1586 #if defined(__lint)
1587 
1588 /* ARGSUSED */
1589 void
1590 repoutsw(int port, uint16_t *addr, int cnt)
1591 {}
1592 
1593 #else   /* __lint */
1594 
1595 #if defined(__amd64)
1596 
1597         ENTRY(repoutsw)
1598         movl    %edx, %ecx
1599         movw    %di, %dx
1600         rep
1601           D16 outsl
1602         ret
1603         SET_SIZE(repoutsw)
1604 
1605 #elif defined(__i386)
1606 
1607         /*
1608          * The arguments and saved registers are on the stack in the
1609          *  following order:
1610          *      |  cnt  |  +16
1611          *      | *addr |  +12
1612          *      | port  |  +8
1613          *      |  eip  |  +4
1614          *      |  esi  |  <-- %esp
1615          * If additional values are pushed onto the stack, make sure
1616          * to adjust the following constants accordingly.
1617          */
1618         .set    PORT, 8
1619         .set    ADDR, 12
1620         .set    COUNT, 16
1621 
1622         ENTRY(repoutsw)
1623         pushl   %esi
1624         movl    PORT(%esp), %edx
1625         movl    ADDR(%esp), %esi
1626         movl    COUNT(%esp), %ecx
1627         rep
1628           D16 outsl
1629         popl    %esi
1630         ret
1631         SET_SIZE(repoutsw)
1632 
1633 #endif  /* __i386 */
1634 #endif  /* __lint */
1635 
1636 
1637 #if defined(__lint)
1638 
1639 /* ARGSUSED */
1640 void
1641 repinsw(int port_addr, uint16_t *addr, int cnt)
1642 {}
1643 
1644 #else   /* __lint */
1645 
1646 #if defined(__amd64)
1647 
1648         ENTRY(repinsw)
1649         movl    %edx, %ecx
1650         movw    %di, %dx
1651         rep
1652           D16 insl
1653         ret
1654         SET_SIZE(repinsw)
1655 
1656 #elif defined(__i386)
1657 
1658         ENTRY(repinsw)
1659         pushl   %edi
1660         movl    PORT(%esp), %edx
1661         movl    ADDR(%esp), %edi
1662         movl    COUNT(%esp), %ecx
1663         rep
1664           D16 insl
1665         popl    %edi
1666         ret
1667         SET_SIZE(repinsw)
1668 
1669 #endif  /* __i386 */
1670 #endif  /* __lint */
1671 
1672 
1673 #if defined(__lint)
1674 
1675 /* ARGSUSED */
1676 void
1677 repinsb(int port, uint8_t *addr, int count)
1678 {}
1679 
1680 #else   /* __lint */
1681 
1682 #if defined(__amd64)
1683 
1684         ENTRY(repinsb)
1685         movl    %edx, %ecx      
1686         movw    %di, %dx
1687         movq    %rsi, %rdi
1688         rep
1689           insb
1690         ret             
1691         SET_SIZE(repinsb)
1692 
1693 #elif defined(__i386)
1694         
1695         /*
1696          * The arguments and saved registers are on the stack in the
1697          *  following order:
1698          *      |  cnt  |  +16
1699          *      | *addr |  +12
1700          *      | port  |  +8
1701          *      |  eip  |  +4
1702          *      |  esi  |  <-- %esp
1703          * If additional values are pushed onto the stack, make sure
1704          * to adjust the following constants accordingly.
1705          */
1706         .set    IO_PORT, 8
1707         .set    IO_ADDR, 12
1708         .set    IO_COUNT, 16
1709 
1710         ENTRY(repinsb)
1711         pushl   %edi
1712         movl    IO_ADDR(%esp), %edi
1713         movl    IO_COUNT(%esp), %ecx
1714         movl    IO_PORT(%esp), %edx
1715         rep
1716           insb
1717         popl    %edi
1718         ret
1719         SET_SIZE(repinsb)
1720 
1721 #endif  /* __i386 */
1722 #endif  /* __lint */
1723 
1724 
1725 /*
1726  * Input a stream of 32-bit words.
1727  * NOTE: count is a DWORD count.
1728  */
1729 #if defined(__lint)
1730 
1731 /* ARGSUSED */
1732 void
1733 repinsd(int port, uint32_t *addr, int count)
1734 {}
1735 
1736 #else   /* __lint */
1737 
1738 #if defined(__amd64)
1739         
1740         ENTRY(repinsd)
1741         movl    %edx, %ecx
1742         movw    %di, %dx
1743         movq    %rsi, %rdi
1744         rep
1745           insl
1746         ret
1747         SET_SIZE(repinsd)
1748 
1749 #elif defined(__i386)
1750 
1751         ENTRY(repinsd)
1752         pushl   %edi
1753         movl    IO_ADDR(%esp), %edi
1754         movl    IO_COUNT(%esp), %ecx
1755         movl    IO_PORT(%esp), %edx
1756         rep
1757           insl
1758         popl    %edi
1759         ret
1760         SET_SIZE(repinsd)
1761 
1762 #endif  /* __i386 */
1763 #endif  /* __lint */
1764 
1765 /*
1766  * Output a stream of bytes
1767  * NOTE: count is a byte count
1768  */
1769 #if defined(__lint)
1770 
1771 /* ARGSUSED */
1772 void
1773 repoutsb(int port, uint8_t *addr, int count)
1774 {}
1775 
1776 #else   /* __lint */
1777 
1778 #if defined(__amd64)
1779 
1780         ENTRY(repoutsb)
1781         movl    %edx, %ecx
1782         movw    %di, %dx
1783         rep
1784           outsb
1785         ret     
1786         SET_SIZE(repoutsb)
1787 
1788 #elif defined(__i386)
1789 
1790         ENTRY(repoutsb)
1791         pushl   %esi
1792         movl    IO_ADDR(%esp), %esi
1793         movl    IO_COUNT(%esp), %ecx
1794         movl    IO_PORT(%esp), %edx
1795         rep
1796           outsb
1797         popl    %esi
1798         ret
1799         SET_SIZE(repoutsb)
1800 
1801 #endif  /* __i386 */    
1802 #endif  /* __lint */
1803 
1804 /*
1805  * Output a stream of 32-bit words
1806  * NOTE: count is a DWORD count
1807  */
1808 #if defined(__lint)
1809 
1810 /* ARGSUSED */
1811 void
1812 repoutsd(int port, uint32_t *addr, int count)
1813 {}
1814 
1815 #else   /* __lint */
1816 
1817 #if defined(__amd64)
1818 
1819         ENTRY(repoutsd)
1820         movl    %edx, %ecx
1821         movw    %di, %dx
1822         rep
1823           outsl
1824         ret     
1825         SET_SIZE(repoutsd)
1826 
1827 #elif defined(__i386)
1828 
1829         ENTRY(repoutsd)
1830         pushl   %esi
1831         movl    IO_ADDR(%esp), %esi
1832         movl    IO_COUNT(%esp), %ecx
1833         movl    IO_PORT(%esp), %edx
1834         rep
1835           outsl
1836         popl    %esi
1837         ret
1838         SET_SIZE(repoutsd)
1839 
1840 #endif  /* __i386 */
1841 #endif  /* __lint */
1842 
1843 /*
1844  * void int3(void)
1845  * void int18(void)
1846  * void int20(void)
1847  * void int_cmci(void)
1848  */
1849 
1850 #if defined(__lint)
1851 
1852 void
1853 int3(void)
1854 {}
1855 
1856 void
1857 int18(void)
1858 {}
1859 
1860 void
1861 int20(void)
1862 {}
1863 
1864 void
1865 int_cmci(void)
1866 {}
1867 
1868 #else   /* __lint */
1869 
1870         ENTRY(int3)
1871         int     $T_BPTFLT
1872         ret
1873         SET_SIZE(int3)
1874 
1875         ENTRY(int18)
1876         int     $T_MCE
1877         ret
1878         SET_SIZE(int18)
1879 
1880         ENTRY(int20)
1881         movl    boothowto, %eax
1882         andl    $RB_DEBUG, %eax
1883         jz      1f
1884 
1885         int     $T_DBGENTR
1886 1:
1887         rep;    ret     /* use 2 byte return instruction when branch target */
1888                         /* AMD Software Optimization Guide - Section 6.2 */
1889         SET_SIZE(int20)
1890 
1891         ENTRY(int_cmci)
1892         int     $T_ENOEXTFLT
1893         ret
1894         SET_SIZE(int_cmci)
1895 
1896 #endif  /* __lint */
1897 
1898 #if defined(__lint)
1899 
1900 /* ARGSUSED */
1901 int
1902 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1903 { return (0); }
1904 
1905 #else   /* __lint */
1906 
1907 #if defined(__amd64)
1908 
1909         ENTRY(scanc)
1910                                         /* rdi == size */
1911                                         /* rsi == cp */
1912                                         /* rdx == table */
1913                                         /* rcx == mask */
1914         addq    %rsi, %rdi              /* end = &cp[size] */
1915 .scanloop:
1916         cmpq    %rdi, %rsi              /* while (cp < end */
1917         jnb     .scandone
1918         movzbq  (%rsi), %r8             /* %r8 = *cp */
1919         incq    %rsi                    /* cp++ */
1920         testb   %cl, (%r8, %rdx)
1921         jz      .scanloop               /*  && (table[*cp] & mask) == 0) */
1922         decq    %rsi                    /* (fix post-increment) */
1923 .scandone:
1924         movl    %edi, %eax
1925         subl    %esi, %eax              /* return (end - cp) */
1926         ret
1927         SET_SIZE(scanc)
1928 
1929 #elif defined(__i386)
1930 
1931         ENTRY(scanc)
1932         pushl   %edi
1933         pushl   %esi
1934         movb    24(%esp), %cl           /* mask = %cl */
1935         movl    16(%esp), %esi          /* cp = %esi */
1936         movl    20(%esp), %edx          /* table = %edx */
1937         movl    %esi, %edi
1938         addl    12(%esp), %edi          /* end = &cp[size]; */
1939 .scanloop:
1940         cmpl    %edi, %esi              /* while (cp < end */
1941         jnb     .scandone
1942         movzbl  (%esi),  %eax           /* %al = *cp */
1943         incl    %esi                    /* cp++ */
1944         movb    (%edx,  %eax), %al      /* %al = table[*cp] */
1945         testb   %al, %cl
1946         jz      .scanloop               /*   && (table[*cp] & mask) == 0) */
1947         dec     %esi                    /* post-incremented */
1948 .scandone:
1949         movl    %edi, %eax
1950         subl    %esi, %eax              /* return (end - cp) */
1951         popl    %esi
1952         popl    %edi
1953         ret
1954         SET_SIZE(scanc)
1955 
1956 #endif  /* __i386 */
1957 #endif  /* __lint */
1958 
1959 /*
1960  * Replacement functions for ones that are normally inlined.
1961  * In addition to the copy in i86.il, they are defined here just in case.
1962  */
1963 
1964 #if defined(__lint)
1965 
1966 ulong_t
1967 intr_clear(void)
1968 { return (0); }
1969 
1970 ulong_t
1971 clear_int_flag(void)
1972 { return (0); }
1973 
1974 #else   /* __lint */
1975 
1976 #if defined(__amd64)
1977 
1978         ENTRY(intr_clear)
1979         ENTRY(clear_int_flag)
1980         pushfq
1981         popq    %rax
1982 #if defined(__xpv)
1983         leaq    xpv_panicking, %rdi
1984         movl    (%rdi), %edi
1985         cmpl    $0, %edi
1986         jne     2f
1987         CLIRET(%rdi, %dl)       /* returns event mask in %dl */
1988         /*
1989          * Synthesize the PS_IE bit from the event mask bit
1990          */
1991         andq    $_BITNOT(PS_IE), %rax
1992         testb   $1, %dl
1993         jnz     1f
1994         orq     $PS_IE, %rax
1995 1:
1996         ret
1997 2:
1998 #endif
1999         CLI(%rdi)
2000         ret
2001         SET_SIZE(clear_int_flag)
2002         SET_SIZE(intr_clear)
2003 
2004 #elif defined(__i386)
2005 
2006         ENTRY(intr_clear)
2007         ENTRY(clear_int_flag)
2008         pushfl
2009         popl    %eax
2010 #if defined(__xpv)
2011         leal    xpv_panicking, %edx
2012         movl    (%edx), %edx
2013         cmpl    $0, %edx
2014         jne     2f
2015         CLIRET(%edx, %cl)       /* returns event mask in %cl */
2016         /*
2017          * Synthesize the PS_IE bit from the event mask bit
2018          */
2019         andl    $_BITNOT(PS_IE), %eax
2020         testb   $1, %cl
2021         jnz     1f
2022         orl     $PS_IE, %eax
2023 1:
2024         ret
2025 2:
2026 #endif
2027         CLI(%edx)
2028         ret
2029         SET_SIZE(clear_int_flag)
2030         SET_SIZE(intr_clear)
2031 
2032 #endif  /* __i386 */
2033 #endif  /* __lint */
2034 
2035 #if defined(__lint)
2036 
2037 struct cpu *
2038 curcpup(void)
2039 { return 0; }
2040 
2041 #else   /* __lint */
2042 
2043 #if defined(__amd64)
2044 
2045         ENTRY(curcpup)
2046         movq    %gs:CPU_SELF, %rax
2047         ret
2048         SET_SIZE(curcpup)
2049 
2050 #elif defined(__i386)
2051 
2052         ENTRY(curcpup)
2053         movl    %gs:CPU_SELF, %eax
2054         ret
2055         SET_SIZE(curcpup)
2056 
2057 #endif  /* __i386 */
2058 #endif  /* __lint */
2059 
2060 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2061  * These functions reverse the byte order of the input parameter and returns
2062  * the result.  This is to convert the byte order from host byte order
2063  * (little endian) to network byte order (big endian), or vice versa.
2064  */
2065 
2066 #if defined(__lint)
2067 
2068 uint64_t
2069 htonll(uint64_t i)
2070 { return (i); }
2071 
2072 uint64_t
2073 ntohll(uint64_t i)
2074 { return (i); }
2075 
2076 uint32_t
2077 htonl(uint32_t i)
2078 { return (i); }
2079 
2080 uint32_t
2081 ntohl(uint32_t i)
2082 { return (i); }
2083 
2084 uint16_t
2085 htons(uint16_t i)
2086 { return (i); }
2087 
2088 uint16_t
2089 ntohs(uint16_t i)
2090 { return (i); }
2091 
2092 #else   /* __lint */
2093 
2094 #if defined(__amd64)
2095 
2096         ENTRY(htonll)
2097         ALTENTRY(ntohll)
2098         movq    %rdi, %rax
2099         bswapq  %rax
2100         ret
2101         SET_SIZE(ntohll)
2102         SET_SIZE(htonll)
2103 
2104         /* XX64 there must be shorter sequences for this */
2105         ENTRY(htonl)
2106         ALTENTRY(ntohl)
2107         movl    %edi, %eax
2108         bswap   %eax
2109         ret
2110         SET_SIZE(ntohl)
2111         SET_SIZE(htonl)
2112 
2113         /* XX64 there must be better sequences for this */
2114         ENTRY(htons)
2115         ALTENTRY(ntohs)
2116         movl    %edi, %eax
2117         bswap   %eax
2118         shrl    $16, %eax
2119         ret
2120         SET_SIZE(ntohs)
2121         SET_SIZE(htons)
2122 
2123 #elif defined(__i386)
2124 
2125         ENTRY(htonll)
2126         ALTENTRY(ntohll)
2127         movl    4(%esp), %edx
2128         movl    8(%esp), %eax
2129         bswap   %edx
2130         bswap   %eax
2131         ret
2132         SET_SIZE(ntohll)
2133         SET_SIZE(htonll)
2134 
2135         ENTRY(htonl)
2136         ALTENTRY(ntohl)
2137         movl    4(%esp), %eax
2138         bswap   %eax
2139         ret
2140         SET_SIZE(ntohl)
2141         SET_SIZE(htonl)
2142 
2143         ENTRY(htons)
2144         ALTENTRY(ntohs)
2145         movl    4(%esp), %eax
2146         bswap   %eax
2147         shrl    $16, %eax
2148         ret
2149         SET_SIZE(ntohs)
2150         SET_SIZE(htons)
2151 
2152 #endif  /* __i386 */
2153 #endif  /* __lint */
2154 
2155 
2156 #if defined(__lint)
2157 
2158 /* ARGSUSED */
2159 void
2160 intr_restore(ulong_t i)
2161 { return; }
2162 
2163 /* ARGSUSED */
2164 void
2165 restore_int_flag(ulong_t i)
2166 { return; }
2167 
2168 #else   /* __lint */
2169 
2170 #if defined(__amd64)
2171 
2172         ENTRY(intr_restore)
2173         ENTRY(restore_int_flag)
2174         testq   $PS_IE, %rdi
2175         jz      1f
2176 #if defined(__xpv)
2177         leaq    xpv_panicking, %rsi
2178         movl    (%rsi), %esi
2179         cmpl    $0, %esi
2180         jne     1f
2181         /*
2182          * Since we're -really- running unprivileged, our attempt
2183          * to change the state of the IF bit will be ignored.
2184          * The virtual IF bit is tweaked by CLI and STI.
2185          */
2186         IE_TO_EVENT_MASK(%rsi, %rdi)
2187 #else
2188         sti
2189 #endif
2190 1:
2191         ret
2192         SET_SIZE(restore_int_flag)
2193         SET_SIZE(intr_restore)
2194 
2195 #elif defined(__i386)
2196 
2197         ENTRY(intr_restore)
2198         ENTRY(restore_int_flag)
2199         testl   $PS_IE, 4(%esp)
2200         jz      1f
2201 #if defined(__xpv)
2202         leal    xpv_panicking, %edx
2203         movl    (%edx), %edx
2204         cmpl    $0, %edx
2205         jne     1f
2206         /*
2207          * Since we're -really- running unprivileged, our attempt
2208          * to change the state of the IF bit will be ignored.
2209          * The virtual IF bit is tweaked by CLI and STI.
2210          */
2211         IE_TO_EVENT_MASK(%edx, 4(%esp))
2212 #else
2213         sti
2214 #endif
2215 1:
2216         ret
2217         SET_SIZE(restore_int_flag)
2218         SET_SIZE(intr_restore)
2219 
2220 #endif  /* __i386 */
2221 #endif  /* __lint */
2222 
2223 #if defined(__lint)
2224 
2225 void
2226 sti(void)
2227 {}
2228 
2229 void
2230 cli(void)
2231 {}
2232 
2233 #else   /* __lint */
2234 
2235         ENTRY(sti)
2236         STI
2237         ret
2238         SET_SIZE(sti)
2239 
2240         ENTRY(cli)
2241 #if defined(__amd64)
2242         CLI(%rax)
2243 #elif defined(__i386)
2244         CLI(%eax)
2245 #endif  /* __i386 */
2246         ret
2247         SET_SIZE(cli)
2248 
2249 #endif  /* __lint */
2250 
2251 #if defined(__lint)
2252 
2253 dtrace_icookie_t
2254 dtrace_interrupt_disable(void)
2255 { return (0); }
2256 
2257 #else   /* __lint */
2258 
2259 #if defined(__amd64)
2260 
2261         ENTRY(dtrace_interrupt_disable)
2262         pushfq
2263         popq    %rax
2264 #if defined(__xpv)
2265         leaq    xpv_panicking, %rdi
2266         movl    (%rdi), %edi
2267         cmpl    $0, %edi
2268         jne     .dtrace_interrupt_disable_done
2269         CLIRET(%rdi, %dl)       /* returns event mask in %dl */
2270         /*
2271          * Synthesize the PS_IE bit from the event mask bit
2272          */
2273         andq    $_BITNOT(PS_IE), %rax
2274         testb   $1, %dl
2275         jnz     .dtrace_interrupt_disable_done
2276         orq     $PS_IE, %rax
2277 #else
2278         CLI(%rdx)
2279 #endif
2280 .dtrace_interrupt_disable_done:
2281         ret
2282         SET_SIZE(dtrace_interrupt_disable)
2283 
2284 #elif defined(__i386)
2285                 
2286         ENTRY(dtrace_interrupt_disable)
2287         pushfl
2288         popl    %eax
2289 #if defined(__xpv)
2290         leal    xpv_panicking, %edx
2291         movl    (%edx), %edx
2292         cmpl    $0, %edx
2293         jne     .dtrace_interrupt_disable_done
2294         CLIRET(%edx, %cl)       /* returns event mask in %cl */
2295         /*
2296          * Synthesize the PS_IE bit from the event mask bit
2297          */
2298         andl    $_BITNOT(PS_IE), %eax
2299         testb   $1, %cl
2300         jnz     .dtrace_interrupt_disable_done
2301         orl     $PS_IE, %eax
2302 #else
2303         CLI(%edx)
2304 #endif
2305 .dtrace_interrupt_disable_done:
2306         ret
2307         SET_SIZE(dtrace_interrupt_disable)
2308 
2309 #endif  /* __i386 */    
2310 #endif  /* __lint */
2311 
2312 #if defined(__lint)
2313 
2314 /*ARGSUSED*/
2315 void
2316 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2317 {}
2318 
2319 #else   /* __lint */
2320 
2321 #if defined(__amd64)
2322 
2323         ENTRY(dtrace_interrupt_enable)
2324         pushq   %rdi
2325         popfq
2326 #if defined(__xpv)
2327         leaq    xpv_panicking, %rdx
2328         movl    (%rdx), %edx
2329         cmpl    $0, %edx
2330         jne     .dtrace_interrupt_enable_done
2331         /*
2332          * Since we're -really- running unprivileged, our attempt
2333          * to change the state of the IF bit will be ignored. The
2334          * virtual IF bit is tweaked by CLI and STI.
2335          */
2336         IE_TO_EVENT_MASK(%rdx, %rdi)
2337 #endif
2338 .dtrace_interrupt_enable_done:
2339         ret
2340         SET_SIZE(dtrace_interrupt_enable)
2341 
2342 #elif defined(__i386)
2343                 
2344         ENTRY(dtrace_interrupt_enable)
2345         movl    4(%esp), %eax
2346         pushl   %eax
2347         popfl
2348 #if defined(__xpv)
2349         leal    xpv_panicking, %edx
2350         movl    (%edx), %edx
2351         cmpl    $0, %edx
2352         jne     .dtrace_interrupt_enable_done
2353         /*
2354          * Since we're -really- running unprivileged, our attempt
2355          * to change the state of the IF bit will be ignored. The
2356          * virtual IF bit is tweaked by CLI and STI.
2357          */
2358         IE_TO_EVENT_MASK(%edx, %eax)
2359 #endif
2360 .dtrace_interrupt_enable_done:
2361         ret
2362         SET_SIZE(dtrace_interrupt_enable)
2363 
2364 #endif  /* __i386 */    
2365 #endif  /* __lint */
2366 
2367 
2368 #if defined(lint)
2369 
2370 void
2371 dtrace_membar_producer(void)
2372 {}
2373 
2374 void
2375 dtrace_membar_consumer(void)
2376 {}
2377 
2378 #else   /* __lint */
2379 
2380         ENTRY(dtrace_membar_producer)
2381         rep;    ret     /* use 2 byte return instruction when branch target */
2382                         /* AMD Software Optimization Guide - Section 6.2 */
2383         SET_SIZE(dtrace_membar_producer)
2384 
2385         ENTRY(dtrace_membar_consumer)
2386         rep;    ret     /* use 2 byte return instruction when branch target */
2387                         /* AMD Software Optimization Guide - Section 6.2 */
2388         SET_SIZE(dtrace_membar_consumer)
2389 
2390 #endif  /* __lint */
2391 
2392 #if defined(__lint)
2393 
2394 kthread_id_t
2395 threadp(void)
2396 { return ((kthread_id_t)0); }
2397 
2398 #else   /* __lint */
2399 
2400 #if defined(__amd64)
2401         
2402         ENTRY(threadp)
2403         movq    %gs:CPU_THREAD, %rax
2404         ret
2405         SET_SIZE(threadp)
2406 
2407 #elif defined(__i386)
2408 
2409         ENTRY(threadp)
2410         movl    %gs:CPU_THREAD, %eax
2411         ret
2412         SET_SIZE(threadp)
2413 
2414 #endif  /* __i386 */
2415 #endif  /* __lint */
2416 
2417 /*
2418  *   Checksum routine for Internet Protocol Headers
2419  */
2420 
2421 #if defined(__lint)
2422 
2423 /* ARGSUSED */
2424 unsigned int
2425 ip_ocsum(
2426         ushort_t *address,      /* ptr to 1st message buffer */
2427         int halfword_count,     /* length of data */
2428         unsigned int sum)       /* partial checksum */
2429 { 
2430         int             i;
2431         unsigned int    psum = 0;       /* partial sum */
2432 
2433         for (i = 0; i < halfword_count; i++, address++) {
2434                 psum += *address;
2435         }
2436 
2437         while ((psum >> 16) != 0) {
2438                 psum = (psum & 0xffff) + (psum >> 16);
2439         }
2440 
2441         psum += sum;
2442 
2443         while ((psum >> 16) != 0) {
2444                 psum = (psum & 0xffff) + (psum >> 16);
2445         }
2446 
2447         return (psum);
2448 }
2449 
2450 #else   /* __lint */
2451 
2452 #if defined(__amd64)
2453 
2454         ENTRY(ip_ocsum)
2455         pushq   %rbp
2456         movq    %rsp, %rbp
2457 #ifdef DEBUG
2458         movq    postbootkernelbase(%rip), %rax
2459         cmpq    %rax, %rdi
2460         jnb     1f
2461         xorl    %eax, %eax
2462         movq    %rdi, %rsi
2463         leaq    .ip_ocsum_panic_msg(%rip), %rdi
2464         call    panic
2465         /*NOTREACHED*/
2466 .ip_ocsum_panic_msg:
2467         .string "ip_ocsum: address 0x%p below kernelbase\n"
2468 1:
2469 #endif
2470         movl    %esi, %ecx      /* halfword_count */
2471         movq    %rdi, %rsi      /* address */
2472                                 /* partial sum in %edx */
2473         xorl    %eax, %eax
2474         testl   %ecx, %ecx
2475         jz      .ip_ocsum_done
2476         testq   $3, %rsi
2477         jnz     .ip_csum_notaligned
2478 .ip_csum_aligned:       /* XX64 opportunities for 8-byte operations? */
2479 .next_iter:
2480         /* XX64 opportunities for prefetch? */
2481         /* XX64 compute csum with 64 bit quantities? */
2482         subl    $32, %ecx
2483         jl      .less_than_32
2484 
2485         addl    0(%rsi), %edx
2486 .only60:
2487         adcl    4(%rsi), %eax
2488 .only56:
2489         adcl    8(%rsi), %edx
2490 .only52:
2491         adcl    12(%rsi), %eax
2492 .only48:
2493         adcl    16(%rsi), %edx
2494 .only44:
2495         adcl    20(%rsi), %eax
2496 .only40:
2497         adcl    24(%rsi), %edx
2498 .only36:
2499         adcl    28(%rsi), %eax
2500 .only32:
2501         adcl    32(%rsi), %edx
2502 .only28:
2503         adcl    36(%rsi), %eax
2504 .only24:
2505         adcl    40(%rsi), %edx
2506 .only20:
2507         adcl    44(%rsi), %eax
2508 .only16:
2509         adcl    48(%rsi), %edx
2510 .only12:
2511         adcl    52(%rsi), %eax
2512 .only8:
2513         adcl    56(%rsi), %edx
2514 .only4:
2515         adcl    60(%rsi), %eax  /* could be adding -1 and -1 with a carry */
2516 .only0:
2517         adcl    $0, %eax        /* could be adding -1 in eax with a carry */
2518         adcl    $0, %eax
2519 
2520         addq    $64, %rsi
2521         testl   %ecx, %ecx
2522         jnz     .next_iter
2523 
2524 .ip_ocsum_done:
2525         addl    %eax, %edx
2526         adcl    $0, %edx
2527         movl    %edx, %eax      /* form a 16 bit checksum by */
2528         shrl    $16, %eax       /* adding two halves of 32 bit checksum */
2529         addw    %dx, %ax
2530         adcw    $0, %ax
2531         andl    $0xffff, %eax
2532         leave
2533         ret
2534 
2535 .ip_csum_notaligned:
2536         xorl    %edi, %edi
2537         movw    (%rsi), %di
2538         addl    %edi, %edx
2539         adcl    $0, %edx
2540         addq    $2, %rsi
2541         decl    %ecx
2542         jmp     .ip_csum_aligned
2543 
2544 .less_than_32:
2545         addl    $32, %ecx
2546         testl   $1, %ecx
2547         jz      .size_aligned
2548         andl    $0xfe, %ecx
2549         movzwl  (%rsi, %rcx, 2), %edi
2550         addl    %edi, %edx
2551         adcl    $0, %edx
2552 .size_aligned:
2553         movl    %ecx, %edi
2554         shrl    $1, %ecx
2555         shl     $1, %edi
2556         subq    $64, %rdi
2557         addq    %rdi, %rsi
2558         leaq    .ip_ocsum_jmptbl(%rip), %rdi
2559         leaq    (%rdi, %rcx, 8), %rdi
2560         xorl    %ecx, %ecx
2561         clc
2562         jmp     *(%rdi)
2563 
2564         .align  8
2565 .ip_ocsum_jmptbl:
2566         .quad   .only0, .only4, .only8, .only12, .only16, .only20
2567         .quad   .only24, .only28, .only32, .only36, .only40, .only44
2568         .quad   .only48, .only52, .only56, .only60
2569         SET_SIZE(ip_ocsum)
2570 
2571 #elif defined(__i386)
2572 
2573         ENTRY(ip_ocsum)
2574         pushl   %ebp
2575         movl    %esp, %ebp
2576         pushl   %ebx
2577         pushl   %esi
2578         pushl   %edi
2579         movl    12(%ebp), %ecx  /* count of half words */
2580         movl    16(%ebp), %edx  /* partial checksum */
2581         movl    8(%ebp), %esi
2582         xorl    %eax, %eax
2583         testl   %ecx, %ecx
2584         jz      .ip_ocsum_done
2585 
2586         testl   $3, %esi
2587         jnz     .ip_csum_notaligned
2588 .ip_csum_aligned:
2589 .next_iter:
2590         subl    $32, %ecx
2591         jl      .less_than_32
2592 
2593         addl    0(%esi), %edx
2594 .only60:
2595         adcl    4(%esi), %eax
2596 .only56:
2597         adcl    8(%esi), %edx
2598 .only52:
2599         adcl    12(%esi), %eax
2600 .only48:
2601         adcl    16(%esi), %edx
2602 .only44:
2603         adcl    20(%esi), %eax
2604 .only40:
2605         adcl    24(%esi), %edx
2606 .only36:
2607         adcl    28(%esi), %eax
2608 .only32:
2609         adcl    32(%esi), %edx
2610 .only28:
2611         adcl    36(%esi), %eax
2612 .only24:
2613         adcl    40(%esi), %edx
2614 .only20:
2615         adcl    44(%esi), %eax
2616 .only16:
2617         adcl    48(%esi), %edx
2618 .only12:
2619         adcl    52(%esi), %eax
2620 .only8:
2621         adcl    56(%esi), %edx
2622 .only4:
2623         adcl    60(%esi), %eax  /* We could be adding -1 and -1 with a carry */
2624 .only0:
2625         adcl    $0, %eax        /* we could be adding -1 in eax with a carry */
2626         adcl    $0, %eax
2627 
2628         addl    $64, %esi
2629         andl    %ecx, %ecx
2630         jnz     .next_iter
2631 
2632 .ip_ocsum_done:
2633         addl    %eax, %edx
2634         adcl    $0, %edx
2635         movl    %edx, %eax      /* form a 16 bit checksum by */
2636         shrl    $16, %eax       /* adding two halves of 32 bit checksum */
2637         addw    %dx, %ax
2638         adcw    $0, %ax
2639         andl    $0xffff, %eax
2640         popl    %edi            /* restore registers */
2641         popl    %esi
2642         popl    %ebx
2643         leave
2644         ret
2645 
2646 .ip_csum_notaligned:
2647         xorl    %edi, %edi
2648         movw    (%esi), %di
2649         addl    %edi, %edx
2650         adcl    $0, %edx
2651         addl    $2, %esi
2652         decl    %ecx
2653         jmp     .ip_csum_aligned
2654 
2655 .less_than_32:
2656         addl    $32, %ecx
2657         testl   $1, %ecx
2658         jz      .size_aligned
2659         andl    $0xfe, %ecx
2660         movzwl  (%esi, %ecx, 2), %edi
2661         addl    %edi, %edx
2662         adcl    $0, %edx
2663 .size_aligned:
2664         movl    %ecx, %edi
2665         shrl    $1, %ecx
2666         shl     $1, %edi
2667         subl    $64, %edi
2668         addl    %edi, %esi
2669         movl    $.ip_ocsum_jmptbl, %edi
2670         lea     (%edi, %ecx, 4), %edi
2671         xorl    %ecx, %ecx
2672         clc
2673         jmp     *(%edi)
2674         SET_SIZE(ip_ocsum)
2675 
2676         .data
2677         .align  4
2678 
2679 .ip_ocsum_jmptbl:
2680         .long   .only0, .only4, .only8, .only12, .only16, .only20
2681         .long   .only24, .only28, .only32, .only36, .only40, .only44
2682         .long   .only48, .only52, .only56, .only60
2683 
2684         
2685 #endif  /* __i386 */            
2686 #endif  /* __lint */
2687 
2688 /*
2689  * multiply two long numbers and yield a u_longlong_t result, callable from C.
2690  * Provided to manipulate hrtime_t values.
2691  */
2692 #if defined(__lint)
2693 
2694 /* result = a * b; */
2695 
2696 /* ARGSUSED */
2697 unsigned long long
2698 mul32(uint_t a, uint_t b)
2699 { return (0); }
2700 
2701 #else   /* __lint */
2702 
2703 #if defined(__amd64)
2704 
2705         ENTRY(mul32)
2706         xorl    %edx, %edx      /* XX64 joe, paranoia? */
2707         movl    %edi, %eax
2708         mull    %esi
2709         shlq    $32, %rdx       
2710         orq     %rdx, %rax
2711         ret
2712         SET_SIZE(mul32)
2713 
2714 #elif defined(__i386)
2715 
2716         ENTRY(mul32)
2717         movl    8(%esp), %eax
2718         movl    4(%esp), %ecx
2719         mull    %ecx
2720         ret
2721         SET_SIZE(mul32)
2722 
2723 #endif  /* __i386 */
2724 #endif  /* __lint */
2725 
2726 #if defined(notused)
2727 #if defined(__lint)
2728 /* ARGSUSED */
2729 void
2730 load_pte64(uint64_t *pte, uint64_t pte_value)
2731 {}
2732 #else   /* __lint */
2733         .globl load_pte64
2734 load_pte64:
2735         movl    4(%esp), %eax
2736         movl    8(%esp), %ecx
2737         movl    12(%esp), %edx
2738         movl    %edx, 4(%eax)
2739         movl    %ecx, (%eax)
2740         ret
2741 #endif  /* __lint */
2742 #endif  /* notused */
2743 
2744 #if defined(__lint)
2745 
2746 /*ARGSUSED*/
2747 void
2748 scan_memory(caddr_t addr, size_t size)
2749 {}
2750 
2751 #else   /* __lint */
2752 
2753 #if defined(__amd64)
2754 
2755         ENTRY(scan_memory)
2756         shrq    $3, %rsi        /* convert %rsi from byte to quadword count */
2757         jz      .scanm_done
2758         movq    %rsi, %rcx      /* move count into rep control register */
2759         movq    %rdi, %rsi      /* move addr into lodsq control reg. */
2760         rep lodsq               /* scan the memory range */
2761 .scanm_done:
2762         rep;    ret     /* use 2 byte return instruction when branch target */
2763                         /* AMD Software Optimization Guide - Section 6.2 */
2764         SET_SIZE(scan_memory)
2765 
2766 #elif defined(__i386)
2767 
2768         ENTRY(scan_memory)
2769         pushl   %ecx
2770         pushl   %esi
2771         movl    16(%esp), %ecx  /* move 2nd arg into rep control register */
2772         shrl    $2, %ecx        /* convert from byte count to word count */
2773         jz      .scanm_done
2774         movl    12(%esp), %esi  /* move 1st arg into lodsw control register */
2775         .byte   0xf3            /* rep prefix.  lame assembler.  sigh. */
2776         lodsl
2777 .scanm_done:
2778         popl    %esi
2779         popl    %ecx
2780         ret
2781         SET_SIZE(scan_memory)
2782 
2783 #endif  /* __i386 */
2784 #endif  /* __lint */
2785 
2786 
2787 #if defined(__lint)
2788 
2789 /*ARGSUSED */
2790 int
2791 lowbit(ulong_t i)
2792 { return (0); }
2793 
2794 #else   /* __lint */
2795 
2796 #if defined(__amd64)
2797 
2798         ENTRY(lowbit)
2799         movl    $-1, %eax
2800         bsfq    %rdi, %rdi
2801         cmovnz  %edi, %eax
2802         incl    %eax
2803         ret
2804         SET_SIZE(lowbit)
2805 
2806 #elif defined(__i386)
2807 
2808         ENTRY(lowbit)
2809         bsfl    4(%esp), %eax
2810         jz      0f
2811         incl    %eax
2812         ret
2813 0:
2814         xorl    %eax, %eax
2815         ret
2816         SET_SIZE(lowbit)
2817 
2818 #endif  /* __i386 */
2819 #endif  /* __lint */
2820 
2821 #if defined(__lint)
2822 
2823 /*ARGSUSED*/
2824 int
2825 highbit(ulong_t i)
2826 { return (0); }
2827 
2828 /*ARGSUSED*/
2829 int
2830 highbit64(uint64_t i)
2831 { return (0); }
2832 
2833 #else   /* __lint */
2834 
2835 #if defined(__amd64)
2836 
2837         ENTRY(highbit)
2838         ALTENTRY(highbit64)
2839         movl    $-1, %eax
2840         bsrq    %rdi, %rdi
2841         cmovnz  %edi, %eax
2842         incl    %eax
2843         ret
2844         SET_SIZE(highbit64)
2845         SET_SIZE(highbit)
2846 
2847 #elif defined(__i386)
2848 
2849         ENTRY(highbit)
2850         bsrl    4(%esp), %eax
2851         jz      0f
2852         incl    %eax
2853         ret
2854 0:
2855         xorl    %eax, %eax
2856         ret    
2857         SET_SIZE(highbit)
2858 
2859         ENTRY(highbit64)
2860         bsrl    8(%esp), %eax
2861         jz      highbit
2862         addl    $33, %eax
2863         ret
2864         SET_SIZE(highbit64)
2865 
2866 #endif  /* __i386 */
2867 #endif  /* __lint */
2868 
2869 #if defined(__lint)
2870 
2871 /*ARGSUSED*/
2872 uint64_t
2873 rdmsr(uint_t r)
2874 { return (0); }
2875 
2876 /*ARGSUSED*/
2877 void
2878 wrmsr(uint_t r, const uint64_t val)
2879 {}
2880 
2881 /*ARGSUSED*/
2882 uint64_t
2883 xrdmsr(uint_t r)
2884 { return (0); }
2885 
2886 /*ARGSUSED*/
2887 void
2888 xwrmsr(uint_t r, const uint64_t val)
2889 {}
2890 
2891 void
2892 invalidate_cache(void)
2893 {}
2894 
2895 /*ARGSUSED*/
2896 uint64_t
2897 get_xcr(uint_t r)
2898 { return (0); }
2899 
2900 /*ARGSUSED*/
2901 void
2902 set_xcr(uint_t r, const uint64_t val)
2903 {}
2904 
2905 #else  /* __lint */
2906 
2907 #define XMSR_ACCESS_VAL         $0x9c5a203a
2908 
2909 #if defined(__amd64)
2910         
2911         ENTRY(rdmsr)
2912         movl    %edi, %ecx
2913         rdmsr
2914         shlq    $32, %rdx
2915         orq     %rdx, %rax
2916         ret
2917         SET_SIZE(rdmsr)
2918 
2919         ENTRY(wrmsr)
2920         movq    %rsi, %rdx
2921         shrq    $32, %rdx
2922         movl    %esi, %eax
2923         movl    %edi, %ecx
2924         wrmsr
2925         ret
2926         SET_SIZE(wrmsr)
2927 
2928         ENTRY(xrdmsr)
2929         pushq   %rbp
2930         movq    %rsp, %rbp
2931         movl    %edi, %ecx
2932         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
2933         rdmsr
2934         shlq    $32, %rdx
2935         orq     %rdx, %rax
2936         leave
2937         ret
2938         SET_SIZE(xrdmsr)
2939 
2940         ENTRY(xwrmsr)
2941         pushq   %rbp
2942         movq    %rsp, %rbp
2943         movl    %edi, %ecx
2944         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
2945         movq    %rsi, %rdx
2946         shrq    $32, %rdx
2947         movl    %esi, %eax
2948         wrmsr
2949         leave
2950         ret
2951         SET_SIZE(xwrmsr)
2952 
2953         ENTRY(get_xcr)
2954         movl    %edi, %ecx
2955         #xgetbv
2956         .byte   0x0f,0x01,0xd0
2957         shlq    $32, %rdx
2958         orq     %rdx, %rax
2959         ret
2960         SET_SIZE(get_xcr)
2961 
2962         ENTRY(set_xcr)
2963         movq    %rsi, %rdx
2964         shrq    $32, %rdx
2965         movl    %esi, %eax
2966         movl    %edi, %ecx
2967         #xsetbv
2968         .byte   0x0f,0x01,0xd1
2969         ret
2970         SET_SIZE(set_xcr)
2971 
2972 #elif defined(__i386)
2973 
2974         ENTRY(rdmsr)
2975         movl    4(%esp), %ecx
2976         rdmsr
2977         ret
2978         SET_SIZE(rdmsr)
2979 
2980         ENTRY(wrmsr)
2981         movl    4(%esp), %ecx
2982         movl    8(%esp), %eax
2983         movl    12(%esp), %edx 
2984         wrmsr
2985         ret
2986         SET_SIZE(wrmsr)
2987 
2988         ENTRY(xrdmsr)
2989         pushl   %ebp
2990         movl    %esp, %ebp
2991         movl    8(%esp), %ecx
2992         pushl   %edi
2993         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
2994         rdmsr
2995         popl    %edi
2996         leave
2997         ret
2998         SET_SIZE(xrdmsr)
2999 
3000         ENTRY(xwrmsr)
3001         pushl   %ebp
3002         movl    %esp, %ebp
3003         movl    8(%esp), %ecx
3004         movl    12(%esp), %eax
3005         movl    16(%esp), %edx 
3006         pushl   %edi
3007         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
3008         wrmsr
3009         popl    %edi
3010         leave
3011         ret
3012         SET_SIZE(xwrmsr)
3013 
3014         ENTRY(get_xcr)
3015         movl    4(%esp), %ecx
3016         #xgetbv
3017         .byte   0x0f,0x01,0xd0
3018         ret
3019         SET_SIZE(get_xcr)
3020 
3021         ENTRY(set_xcr)
3022         movl    4(%esp), %ecx
3023         movl    8(%esp), %eax
3024         movl    12(%esp), %edx
3025         #xsetbv
3026         .byte   0x0f,0x01,0xd1
3027         ret
3028         SET_SIZE(set_xcr)
3029 
3030 #endif  /* __i386 */
3031 
3032         ENTRY(invalidate_cache)
3033         wbinvd
3034         ret
3035         SET_SIZE(invalidate_cache)
3036 
3037 #endif  /* __lint */
3038 
3039 #if defined(__lint)
3040 
3041 /*ARGSUSED*/
3042 void
3043 getcregs(struct cregs *crp)
3044 {}
3045 
3046 #else   /* __lint */
3047 
3048 #if defined(__amd64)
3049 
3050         ENTRY_NP(getcregs)
3051 #if defined(__xpv)
3052         /*
3053          * Only a few of the hardware control registers or descriptor tables
3054          * are directly accessible to us, so just zero the structure.
3055          *
3056          * XXPV Perhaps it would be helpful for the hypervisor to return
3057          *      virtualized versions of these for post-mortem use.
3058          *      (Need to reevaluate - perhaps it already does!)
3059          */
3060         pushq   %rdi            /* save *crp */
3061         movq    $CREGSZ, %rsi
3062         call    bzero
3063         popq    %rdi
3064 
3065         /*
3066          * Dump what limited information we can
3067          */
3068         movq    %cr0, %rax
3069         movq    %rax, CREG_CR0(%rdi)    /* cr0 */
3070         movq    %cr2, %rax
3071         movq    %rax, CREG_CR2(%rdi)    /* cr2 */
3072         movq    %cr3, %rax
3073         movq    %rax, CREG_CR3(%rdi)    /* cr3 */
3074         movq    %cr4, %rax
3075         movq    %rax, CREG_CR4(%rdi)    /* cr4 */
3076 
3077 #else   /* __xpv */
3078 
3079 #define GETMSR(r, off, d)       \
3080         movl    $r, %ecx;       \
3081         rdmsr;                  \
3082         movl    %eax, off(d);   \
3083         movl    %edx, off+4(d)
3084 
3085         xorl    %eax, %eax
3086         movq    %rax, CREG_GDT+8(%rdi)
3087         sgdt    CREG_GDT(%rdi)          /* 10 bytes */
3088         movq    %rax, CREG_IDT+8(%rdi)
3089         sidt    CREG_IDT(%rdi)          /* 10 bytes */
3090         movq    %rax, CREG_LDT(%rdi)
3091         sldt    CREG_LDT(%rdi)          /* 2 bytes */
3092         movq    %rax, CREG_TASKR(%rdi)
3093         str     CREG_TASKR(%rdi)        /* 2 bytes */
3094         movq    %cr0, %rax
3095         movq    %rax, CREG_CR0(%rdi)    /* cr0 */
3096         movq    %cr2, %rax
3097         movq    %rax, CREG_CR2(%rdi)    /* cr2 */
3098         movq    %cr3, %rax
3099         movq    %rax, CREG_CR3(%rdi)    /* cr3 */
3100         movq    %cr4, %rax
3101         movq    %rax, CREG_CR4(%rdi)    /* cr4 */
3102         movq    %cr8, %rax
3103         movq    %rax, CREG_CR8(%rdi)    /* cr8 */
3104         GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3105         GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3106 #endif  /* __xpv */
3107         ret
3108         SET_SIZE(getcregs)
3109 
3110 #undef GETMSR
3111 
3112 #elif defined(__i386)
3113 
3114         ENTRY_NP(getcregs)
3115 #if defined(__xpv)
3116         /*
3117          * Only a few of the hardware control registers or descriptor tables
3118          * are directly accessible to us, so just zero the structure.
3119          *
3120          * XXPV Perhaps it would be helpful for the hypervisor to return
3121          *      virtualized versions of these for post-mortem use.
3122          *      (Need to reevaluate - perhaps it already does!)
3123          */
3124         movl    4(%esp), %edx
3125         pushl   $CREGSZ
3126         pushl   %edx
3127         call    bzero
3128         addl    $8, %esp
3129         movl    4(%esp), %edx
3130 
3131         /*
3132          * Dump what limited information we can
3133          */
3134         movl    %cr0, %eax
3135         movl    %eax, CREG_CR0(%edx)    /* cr0 */
3136         movl    %cr2, %eax
3137         movl    %eax, CREG_CR2(%edx)    /* cr2 */
3138         movl    %cr3, %eax
3139         movl    %eax, CREG_CR3(%edx)    /* cr3 */
3140         movl    %cr4, %eax
3141         movl    %eax, CREG_CR4(%edx)    /* cr4 */
3142 
3143 #else   /* __xpv */
3144 
3145         movl    4(%esp), %edx
3146         movw    $0, CREG_GDT+6(%edx)
3147         movw    $0, CREG_IDT+6(%edx)
3148         sgdt    CREG_GDT(%edx)          /* gdt */
3149         sidt    CREG_IDT(%edx)          /* idt */
3150         sldt    CREG_LDT(%edx)          /* ldt */
3151         str     CREG_TASKR(%edx)        /* task */
3152         movl    %cr0, %eax
3153         movl    %eax, CREG_CR0(%edx)    /* cr0 */
3154         movl    %cr2, %eax
3155         movl    %eax, CREG_CR2(%edx)    /* cr2 */
3156         movl    %cr3, %eax
3157         movl    %eax, CREG_CR3(%edx)    /* cr3 */
3158         bt      $X86FSET_LARGEPAGE, x86_featureset
3159         jnc     .nocr4
3160         movl    %cr4, %eax
3161         movl    %eax, CREG_CR4(%edx)    /* cr4 */
3162         jmp     .skip
3163 .nocr4:
3164         movl    $0, CREG_CR4(%edx)
3165 .skip:
3166 #endif
3167         ret
3168         SET_SIZE(getcregs)
3169 
3170 #endif  /* __i386 */
3171 #endif  /* __lint */
3172 
3173 
3174 /*
3175  * A panic trigger is a word which is updated atomically and can only be set
3176  * once.  We atomically store 0xDEFACEDD and load the old value.  If the
3177  * previous value was 0, we succeed and return 1; otherwise return 0.
3178  * This allows a partially corrupt trigger to still trigger correctly.  DTrace
3179  * has its own version of this function to allow it to panic correctly from
3180  * probe context.
3181  */
3182 #if defined(__lint)
3183 
3184 /*ARGSUSED*/
3185 int
3186 panic_trigger(int *tp)
3187 { return (0); }
3188 
3189 /*ARGSUSED*/
3190 int
3191 dtrace_panic_trigger(int *tp)
3192 { return (0); }
3193 
3194 #else   /* __lint */
3195 
3196 #if defined(__amd64)
3197 
3198         ENTRY_NP(panic_trigger)
3199         xorl    %eax, %eax
3200         movl    $0xdefacedd, %edx
3201         lock
3202           xchgl %edx, (%rdi)
3203         cmpl    $0, %edx
3204         je      0f 
3205         movl    $0, %eax
3206         ret
3207 0:      movl    $1, %eax
3208         ret
3209         SET_SIZE(panic_trigger)
3210         
3211         ENTRY_NP(dtrace_panic_trigger)
3212         xorl    %eax, %eax
3213         movl    $0xdefacedd, %edx
3214         lock
3215           xchgl %edx, (%rdi)
3216         cmpl    $0, %edx
3217         je      0f
3218         movl    $0, %eax
3219         ret
3220 0:      movl    $1, %eax
3221         ret
3222         SET_SIZE(dtrace_panic_trigger)
3223 
3224 #elif defined(__i386)
3225 
3226         ENTRY_NP(panic_trigger)
3227         movl    4(%esp), %edx           / %edx = address of trigger
3228         movl    $0xdefacedd, %eax       / %eax = 0xdefacedd
3229         lock                            / assert lock
3230         xchgl %eax, (%edx)              / exchange %eax and the trigger
3231         cmpl    $0, %eax                / if (%eax == 0x0)
3232         je      0f                      /   return (1);
3233         movl    $0, %eax                / else
3234         ret                             /   return (0);
3235 0:      movl    $1, %eax
3236         ret
3237         SET_SIZE(panic_trigger)
3238 
3239         ENTRY_NP(dtrace_panic_trigger)
3240         movl    4(%esp), %edx           / %edx = address of trigger
3241         movl    $0xdefacedd, %eax       / %eax = 0xdefacedd
3242         lock                            / assert lock
3243         xchgl %eax, (%edx)              / exchange %eax and the trigger
3244         cmpl    $0, %eax                / if (%eax == 0x0)
3245         je      0f                      /   return (1);
3246         movl    $0, %eax                / else
3247         ret                             /   return (0);
3248 0:      movl    $1, %eax
3249         ret
3250         SET_SIZE(dtrace_panic_trigger)
3251 
3252 #endif  /* __i386 */
3253 #endif  /* __lint */
3254 
3255 /*
3256  * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3257  * into the panic code implemented in panicsys().  vpanic() is responsible
3258  * for passing through the format string and arguments, and constructing a
3259  * regs structure on the stack into which it saves the current register
3260  * values.  If we are not dying due to a fatal trap, these registers will
3261  * then be preserved in panicbuf as the current processor state.  Before
3262  * invoking panicsys(), vpanic() activates the first panic trigger (see
3263  * common/os/panic.c) and switches to the panic_stack if successful.  Note that
3264  * DTrace takes a slightly different panic path if it must panic from probe
3265  * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
3266  * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3267  * branches back into vpanic().
3268  */
3269 #if defined(__lint)
3270 
3271 /*ARGSUSED*/
3272 void
3273 vpanic(const char *format, va_list alist)
3274 {}
3275 
3276 /*ARGSUSED*/
3277 void
3278 dtrace_vpanic(const char *format, va_list alist)
3279 {}
3280 
3281 #else   /* __lint */
3282 
3283 #if defined(__amd64)
3284 
3285         ENTRY_NP(vpanic)                        /* Initial stack layout: */
3286         
3287         pushq   %rbp                            /* | %rip |     0x60    */
3288         movq    %rsp, %rbp                      /* | %rbp |     0x58    */
3289         pushfq                                  /* | rfl  |     0x50    */
3290         pushq   %r11                            /* | %r11 |     0x48    */
3291         pushq   %r10                            /* | %r10 |     0x40    */
3292         pushq   %rbx                            /* | %rbx |     0x38    */
3293         pushq   %rax                            /* | %rax |     0x30    */
3294         pushq   %r9                             /* | %r9  |     0x28    */
3295         pushq   %r8                             /* | %r8  |     0x20    */
3296         pushq   %rcx                            /* | %rcx |     0x18    */
3297         pushq   %rdx                            /* | %rdx |     0x10    */
3298         pushq   %rsi                            /* | %rsi |     0x8 alist */
3299         pushq   %rdi                            /* | %rdi |     0x0 format */
3300 
3301         movq    %rsp, %rbx                      /* %rbx = current %rsp */
3302 
3303         leaq    panic_quiesce(%rip), %rdi       /* %rdi = &panic_quiesce */
3304         call    panic_trigger                   /* %eax = panic_trigger() */
3305 
3306 vpanic_common:
3307         /*
3308          * The panic_trigger result is in %eax from the call above, and
3309          * dtrace_panic places it in %eax before branching here.
3310          * The rdmsr instructions that follow below will clobber %eax so
3311          * we stash the panic_trigger result in %r11d.
3312          */
3313         movl    %eax, %r11d
3314         cmpl    $0, %r11d
3315         je      0f
3316 
3317         /*
3318          * If panic_trigger() was successful, we are the first to initiate a
3319          * panic: we now switch to the reserved panic_stack before continuing.
3320          */
3321         leaq    panic_stack(%rip), %rsp
3322         addq    $PANICSTKSIZE, %rsp
3323 0:      subq    $REGSIZE, %rsp
3324         /*
3325          * Now that we've got everything set up, store the register values as
3326          * they were when we entered vpanic() to the designated location in
3327          * the regs structure we allocated on the stack.
3328          */
3329         movq    0x0(%rbx), %rcx
3330         movq    %rcx, REGOFF_RDI(%rsp)
3331         movq    0x8(%rbx), %rcx
3332         movq    %rcx, REGOFF_RSI(%rsp)
3333         movq    0x10(%rbx), %rcx
3334         movq    %rcx, REGOFF_RDX(%rsp)
3335         movq    0x18(%rbx), %rcx
3336         movq    %rcx, REGOFF_RCX(%rsp)
3337         movq    0x20(%rbx), %rcx
3338 
3339         movq    %rcx, REGOFF_R8(%rsp)
3340         movq    0x28(%rbx), %rcx
3341         movq    %rcx, REGOFF_R9(%rsp)
3342         movq    0x30(%rbx), %rcx
3343         movq    %rcx, REGOFF_RAX(%rsp)
3344         movq    0x38(%rbx), %rcx
3345         movq    %rcx, REGOFF_RBX(%rsp)
3346         movq    0x58(%rbx), %rcx
3347 
3348         movq    %rcx, REGOFF_RBP(%rsp)
3349         movq    0x40(%rbx), %rcx
3350         movq    %rcx, REGOFF_R10(%rsp)
3351         movq    0x48(%rbx), %rcx
3352         movq    %rcx, REGOFF_R11(%rsp)
3353         movq    %r12, REGOFF_R12(%rsp)
3354 
3355         movq    %r13, REGOFF_R13(%rsp)
3356         movq    %r14, REGOFF_R14(%rsp)
3357         movq    %r15, REGOFF_R15(%rsp)
3358 
3359         xorl    %ecx, %ecx
3360         movw    %ds, %cx
3361         movq    %rcx, REGOFF_DS(%rsp)
3362         movw    %es, %cx
3363         movq    %rcx, REGOFF_ES(%rsp)
3364         movw    %fs, %cx
3365         movq    %rcx, REGOFF_FS(%rsp)
3366         movw    %gs, %cx
3367         movq    %rcx, REGOFF_GS(%rsp)
3368 
3369         movq    $0, REGOFF_TRAPNO(%rsp)
3370 
3371         movq    $0, REGOFF_ERR(%rsp)
3372         leaq    vpanic(%rip), %rcx
3373         movq    %rcx, REGOFF_RIP(%rsp)
3374         movw    %cs, %cx
3375         movzwq  %cx, %rcx
3376         movq    %rcx, REGOFF_CS(%rsp)
3377         movq    0x50(%rbx), %rcx
3378         movq    %rcx, REGOFF_RFL(%rsp)
3379         movq    %rbx, %rcx
3380         addq    $0x60, %rcx
3381         movq    %rcx, REGOFF_RSP(%rsp)
3382         movw    %ss, %cx
3383         movzwq  %cx, %rcx
3384         movq    %rcx, REGOFF_SS(%rsp)
3385 
3386         /*
3387          * panicsys(format, alist, rp, on_panic_stack) 
3388          */     
3389         movq    REGOFF_RDI(%rsp), %rdi          /* format */
3390         movq    REGOFF_RSI(%rsp), %rsi          /* alist */
3391         movq    %rsp, %rdx                      /* struct regs */
3392         movl    %r11d, %ecx                     /* on_panic_stack */
3393         call    panicsys
3394         addq    $REGSIZE, %rsp
3395         popq    %rdi
3396         popq    %rsi
3397         popq    %rdx
3398         popq    %rcx
3399         popq    %r8
3400         popq    %r9
3401         popq    %rax
3402         popq    %rbx
3403         popq    %r10
3404         popq    %r11
3405         popfq
3406         leave
3407         ret
3408         SET_SIZE(vpanic)
3409 
3410         ENTRY_NP(dtrace_vpanic)                 /* Initial stack layout: */
3411 
3412         pushq   %rbp                            /* | %rip |     0x60    */
3413         movq    %rsp, %rbp                      /* | %rbp |     0x58    */
3414         pushfq                                  /* | rfl  |     0x50    */
3415         pushq   %r11                            /* | %r11 |     0x48    */
3416         pushq   %r10                            /* | %r10 |     0x40    */
3417         pushq   %rbx                            /* | %rbx |     0x38    */
3418         pushq   %rax                            /* | %rax |     0x30    */
3419         pushq   %r9                             /* | %r9  |     0x28    */
3420         pushq   %r8                             /* | %r8  |     0x20    */
3421         pushq   %rcx                            /* | %rcx |     0x18    */
3422         pushq   %rdx                            /* | %rdx |     0x10    */
3423         pushq   %rsi                            /* | %rsi |     0x8 alist */
3424         pushq   %rdi                            /* | %rdi |     0x0 format */
3425 
3426         movq    %rsp, %rbx                      /* %rbx = current %rsp */
3427 
3428         leaq    panic_quiesce(%rip), %rdi       /* %rdi = &panic_quiesce */
3429         call    dtrace_panic_trigger    /* %eax = dtrace_panic_trigger() */
3430         jmp     vpanic_common
3431 
3432         SET_SIZE(dtrace_vpanic)
3433 
3434 #elif defined(__i386)
3435 
3436         ENTRY_NP(vpanic)                        / Initial stack layout:
3437 
3438         pushl   %ebp                            / | %eip | 20
3439         movl    %esp, %ebp                      / | %ebp | 16
3440         pushl   %eax                            / | %eax | 12
3441         pushl   %ebx                            / | %ebx |  8
3442         pushl   %ecx                            / | %ecx |  4
3443         pushl   %edx                            / | %edx |  0
3444 
3445         movl    %esp, %ebx                      / %ebx = current stack pointer
3446 
3447         lea     panic_quiesce, %eax             / %eax = &panic_quiesce
3448         pushl   %eax                            / push &panic_quiesce
3449         call    panic_trigger                   / %eax = panic_trigger()
3450         addl    $4, %esp                        / reset stack pointer
3451 
3452 vpanic_common:
3453         cmpl    $0, %eax                        / if (%eax == 0)
3454         je      0f                              /   goto 0f;
3455 
3456         /*
3457          * If panic_trigger() was successful, we are the first to initiate a
3458          * panic: we now switch to the reserved panic_stack before continuing.
3459          */
3460         lea     panic_stack, %esp               / %esp  = panic_stack
3461         addl    $PANICSTKSIZE, %esp             / %esp += PANICSTKSIZE
3462 
3463 0:      subl    $REGSIZE, %esp                  / allocate struct regs
3464 
3465         /*
3466          * Now that we've got everything set up, store the register values as
3467          * they were when we entered vpanic() to the designated location in
3468          * the regs structure we allocated on the stack. 
3469          */
3470 #if !defined(__GNUC_AS__)
3471         movw    %gs, %edx
3472         movl    %edx, REGOFF_GS(%esp)
3473         movw    %fs, %edx
3474         movl    %edx, REGOFF_FS(%esp)
3475         movw    %es, %edx
3476         movl    %edx, REGOFF_ES(%esp)
3477         movw    %ds, %edx
3478         movl    %edx, REGOFF_DS(%esp)
3479 #else   /* __GNUC_AS__ */
3480         mov     %gs, %edx
3481         mov     %edx, REGOFF_GS(%esp)
3482         mov     %fs, %edx
3483         mov     %edx, REGOFF_FS(%esp)
3484         mov     %es, %edx
3485         mov     %edx, REGOFF_ES(%esp)
3486         mov     %ds, %edx
3487         mov     %edx, REGOFF_DS(%esp)
3488 #endif  /* __GNUC_AS__ */
3489         movl    %edi, REGOFF_EDI(%esp)
3490         movl    %esi, REGOFF_ESI(%esp)
3491         movl    16(%ebx), %ecx
3492         movl    %ecx, REGOFF_EBP(%esp)
3493         movl    %ebx, %ecx
3494         addl    $20, %ecx
3495         movl    %ecx, REGOFF_ESP(%esp)
3496         movl    8(%ebx), %ecx
3497         movl    %ecx, REGOFF_EBX(%esp)
3498         movl    0(%ebx), %ecx
3499         movl    %ecx, REGOFF_EDX(%esp)
3500         movl    4(%ebx), %ecx
3501         movl    %ecx, REGOFF_ECX(%esp)
3502         movl    12(%ebx), %ecx
3503         movl    %ecx, REGOFF_EAX(%esp)
3504         movl    $0, REGOFF_TRAPNO(%esp)
3505         movl    $0, REGOFF_ERR(%esp)
3506         lea     vpanic, %ecx
3507         movl    %ecx, REGOFF_EIP(%esp)
3508 #if !defined(__GNUC_AS__)
3509         movw    %cs, %edx
3510 #else   /* __GNUC_AS__ */
3511         mov     %cs, %edx
3512 #endif  /* __GNUC_AS__ */
3513         movl    %edx, REGOFF_CS(%esp)
3514         pushfl
3515         popl    %ecx
3516 #if defined(__xpv)
3517         /*
3518          * Synthesize the PS_IE bit from the event mask bit
3519          */
3520         CURTHREAD(%edx)
3521         KPREEMPT_DISABLE(%edx)
3522         EVENT_MASK_TO_IE(%edx, %ecx)
3523         CURTHREAD(%edx)
3524         KPREEMPT_ENABLE_NOKP(%edx)
3525 #endif
3526         movl    %ecx, REGOFF_EFL(%esp)
3527         movl    $0, REGOFF_UESP(%esp)
3528 #if !defined(__GNUC_AS__)
3529         movw    %ss, %edx
3530 #else   /* __GNUC_AS__ */
3531         mov     %ss, %edx
3532 #endif  /* __GNUC_AS__ */
3533         movl    %edx, REGOFF_SS(%esp)
3534 
3535         movl    %esp, %ecx                      / %ecx = &regs
3536         pushl   %eax                            / push on_panic_stack
3537         pushl   %ecx                            / push &regs
3538         movl    12(%ebp), %ecx                  / %ecx = alist
3539         pushl   %ecx                            / push alist
3540         movl    8(%ebp), %ecx                   / %ecx = format
3541         pushl   %ecx                            / push format
3542         call    panicsys                        / panicsys();
3543         addl    $16, %esp                       / pop arguments
3544 
3545         addl    $REGSIZE, %esp
3546         popl    %edx
3547         popl    %ecx
3548         popl    %ebx
3549         popl    %eax
3550         leave
3551         ret
3552         SET_SIZE(vpanic)
3553 
3554         ENTRY_NP(dtrace_vpanic)                 / Initial stack layout:
3555 
3556         pushl   %ebp                            / | %eip | 20
3557         movl    %esp, %ebp                      / | %ebp | 16
3558         pushl   %eax                            / | %eax | 12
3559         pushl   %ebx                            / | %ebx |  8
3560         pushl   %ecx                            / | %ecx |  4
3561         pushl   %edx                            / | %edx |  0
3562 
3563         movl    %esp, %ebx                      / %ebx = current stack pointer
3564 
3565         lea     panic_quiesce, %eax             / %eax = &panic_quiesce
3566         pushl   %eax                            / push &panic_quiesce
3567         call    dtrace_panic_trigger            / %eax = dtrace_panic_trigger()
3568         addl    $4, %esp                        / reset stack pointer
3569         jmp     vpanic_common                   / jump back to common code
3570 
3571         SET_SIZE(dtrace_vpanic)
3572 
3573 #endif  /* __i386 */
3574 #endif  /* __lint */
3575 
3576 #if defined(__lint)
3577 
3578 void
3579 hres_tick(void)
3580 {}
3581 
3582 int64_t timedelta;
3583 hrtime_t hrtime_base;
3584 
3585 #else   /* __lint */
3586 
3587         DGDEF3(timedelta, 8, 8)
3588         .long   0, 0
3589 
3590         /*
3591          * initialized to a non zero value to make pc_gethrtime()
3592          * work correctly even before clock is initialized
3593          */
3594         DGDEF3(hrtime_base, 8, 8)
3595         .long   _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3596 
3597         DGDEF3(adj_shift, 4, 4)
3598         .long   ADJ_SHIFT
3599 
3600 #if defined(__amd64)
3601 
3602         ENTRY_NP(hres_tick)
3603         pushq   %rbp
3604         movq    %rsp, %rbp
3605 
3606         /*
3607          * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3608          * hres_last_tick can only be modified while holding CLOCK_LOCK).
3609          * At worst, performing this now instead of under CLOCK_LOCK may
3610          * introduce some jitter in pc_gethrestime().
3611          */
3612         call    *gethrtimef(%rip)
3613         movq    %rax, %r8
3614 
3615         leaq    hres_lock(%rip), %rax
3616         movb    $-1, %dl
3617 .CL1:
3618         xchgb   %dl, (%rax)
3619         testb   %dl, %dl
3620         jz      .CL3                    /* got it */
3621 .CL2:
3622         cmpb    $0, (%rax)              /* possible to get lock? */
3623         pause
3624         jne     .CL2
3625         jmp     .CL1                    /* yes, try again */
3626 .CL3:
3627         /*
3628          * compute the interval since last time hres_tick was called
3629          * and adjust hrtime_base and hrestime accordingly
3630          * hrtime_base is an 8 byte value (in nsec), hrestime is
3631          * a timestruc_t (sec, nsec)
3632          */
3633         leaq    hres_last_tick(%rip), %rax
3634         movq    %r8, %r11
3635         subq    (%rax), %r8
3636         addq    %r8, hrtime_base(%rip)  /* add interval to hrtime_base */
3637         addq    %r8, hrestime+8(%rip)   /* add interval to hrestime.tv_nsec */
3638         /*
3639          * Now that we have CLOCK_LOCK, we can update hres_last_tick
3640          */     
3641         movq    %r11, (%rax)    
3642 
3643         call    __adj_hrestime
3644 
3645         /*
3646          * release the hres_lock
3647          */
3648         incl    hres_lock(%rip)
3649         leave
3650         ret
3651         SET_SIZE(hres_tick)
3652         
3653 #elif defined(__i386)
3654 
3655         ENTRY_NP(hres_tick)
3656         pushl   %ebp
3657         movl    %esp, %ebp
3658         pushl   %esi
3659         pushl   %ebx
3660 
3661         /*
3662          * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3663          * hres_last_tick can only be modified while holding CLOCK_LOCK).
3664          * At worst, performing this now instead of under CLOCK_LOCK may
3665          * introduce some jitter in pc_gethrestime().
3666          */
3667         call    *gethrtimef
3668         movl    %eax, %ebx
3669         movl    %edx, %esi
3670 
3671         movl    $hres_lock, %eax
3672         movl    $-1, %edx
3673 .CL1:
3674         xchgb   %dl, (%eax)
3675         testb   %dl, %dl
3676         jz      .CL3                    / got it
3677 .CL2:
3678         cmpb    $0, (%eax)              / possible to get lock?
3679         pause
3680         jne     .CL2
3681         jmp     .CL1                    / yes, try again
3682 .CL3:
3683         /*
3684          * compute the interval since last time hres_tick was called
3685          * and adjust hrtime_base and hrestime accordingly
3686          * hrtime_base is an 8 byte value (in nsec), hrestime is
3687          * timestruc_t (sec, nsec)
3688          */
3689 
3690         lea     hres_last_tick, %eax
3691 
3692         movl    %ebx, %edx
3693         movl    %esi, %ecx
3694 
3695         subl    (%eax), %edx
3696         sbbl    4(%eax), %ecx
3697 
3698         addl    %edx, hrtime_base       / add interval to hrtime_base
3699         adcl    %ecx, hrtime_base+4
3700 
3701         addl    %edx, hrestime+4        / add interval to hrestime.tv_nsec
3702 
3703         /
3704         / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3705         /
3706         movl    %ebx, (%eax)
3707         movl    %esi,  4(%eax)
3708 
3709         / get hrestime at this moment. used as base for pc_gethrestime
3710         /
3711         / Apply adjustment, if any
3712         /
3713         / #define HRES_ADJ      (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3714         / (max_hres_adj)
3715         /
3716         / void
3717         / adj_hrestime()
3718         / {
3719         /       long long adj;
3720         /
3721         /       if (hrestime_adj == 0)
3722         /               adj = 0;
3723         /       else if (hrestime_adj > 0) {
3724         /               if (hrestime_adj < HRES_ADJ)
3725         /                       adj = hrestime_adj;
3726         /               else
3727         /                       adj = HRES_ADJ;
3728         /       }
3729         /       else {
3730         /               if (hrestime_adj < -(HRES_ADJ))
3731         /                       adj = -(HRES_ADJ);
3732         /               else
3733         /                       adj = hrestime_adj;
3734         /       }
3735         /
3736         /       timedelta -= adj;
3737         /       hrestime_adj = timedelta;
3738         /       hrestime.tv_nsec += adj;
3739         /
3740         /       while (hrestime.tv_nsec >= NANOSEC) {
3741         /               one_sec++;
3742         /               hrestime.tv_sec++;
3743         /               hrestime.tv_nsec -= NANOSEC;
3744         /       }
3745         / }
3746 __adj_hrestime:
3747         movl    hrestime_adj, %esi      / if (hrestime_adj == 0)
3748         movl    hrestime_adj+4, %edx
3749         andl    %esi, %esi
3750         jne     .CL4                    / no
3751         andl    %edx, %edx
3752         jne     .CL4                    / no
3753         subl    %ecx, %ecx              / yes, adj = 0;
3754         subl    %edx, %edx
3755         jmp     .CL5
3756 .CL4:
3757         subl    %ecx, %ecx
3758         subl    %eax, %eax
3759         subl    %esi, %ecx
3760         sbbl    %edx, %eax
3761         andl    %eax, %eax              / if (hrestime_adj > 0)
3762         jge     .CL6
3763 
3764         / In the following comments, HRES_ADJ is used, while in the code
3765         / max_hres_adj is used.
3766         /
3767         / The test for "hrestime_adj < HRES_ADJ" is complicated because
3768         / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits.  We rely
3769         / on the logical equivalence of:
3770         /
3771         /       !(hrestime_adj < HRES_ADJ)
3772         /
3773         / and the two step sequence:
3774         /
3775         /       (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3776         /
3777         / which computes whether or not the least significant 32-bits
3778         / of hrestime_adj is greater than HRES_ADJ, followed by:
3779         /
3780         /       Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3781         /
3782         / which generates a carry whenever step 1 is true or the most
3783         / significant long of the longlong hrestime_adj is non-zero.
3784 
3785         movl    max_hres_adj, %ecx      / hrestime_adj is positive
3786         subl    %esi, %ecx
3787         movl    %edx, %eax
3788         adcl    $-1, %eax
3789         jnc     .CL7
3790         movl    max_hres_adj, %ecx      / adj = HRES_ADJ;
3791         subl    %edx, %edx
3792         jmp     .CL5
3793 
3794         / The following computation is similar to the one above.
3795         /
3796         / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3797         / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits.  We rely
3798         / on the logical equivalence of:
3799         /
3800         /       (hrestime_adj > -HRES_ADJ)
3801         /
3802         / and the two step sequence:
3803         /
3804         /       (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3805         /
3806         / which means the least significant 32-bits of hrestime_adj is
3807         / greater than -HRES_ADJ, followed by:
3808         /
3809         /       Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3810         /
3811         / which generates a carry only when step 1 is true and the most
3812         / significant long of the longlong hrestime_adj is -1.
3813 
3814 .CL6:                                   / hrestime_adj is negative
3815         movl    %esi, %ecx
3816         addl    max_hres_adj, %ecx
3817         movl    %edx, %eax
3818         adcl    $0, %eax
3819         jc      .CL7
3820         xor     %ecx, %ecx
3821         subl    max_hres_adj, %ecx      / adj = -(HRES_ADJ);
3822         movl    $-1, %edx
3823         jmp     .CL5
3824 .CL7:
3825         movl    %esi, %ecx              / adj = hrestime_adj;
3826 .CL5:
3827         movl    timedelta, %esi
3828         subl    %ecx, %esi
3829         movl    timedelta+4, %eax
3830         sbbl    %edx, %eax
3831         movl    %esi, timedelta
3832         movl    %eax, timedelta+4       / timedelta -= adj;
3833         movl    %esi, hrestime_adj
3834         movl    %eax, hrestime_adj+4    / hrestime_adj = timedelta;
3835         addl    hrestime+4, %ecx
3836 
3837         movl    %ecx, %eax              / eax = tv_nsec
3838 1:
3839         cmpl    $NANOSEC, %eax          / if ((unsigned long)tv_nsec >= NANOSEC)
3840         jb      .CL8                    / no
3841         incl    one_sec                 / yes,  one_sec++;
3842         incl    hrestime                / hrestime.tv_sec++;
3843         addl    $-NANOSEC, %eax         / tv_nsec -= NANOSEC
3844         jmp     1b                      / check for more seconds
3845 
3846 .CL8:
3847         movl    %eax, hrestime+4        / store final into hrestime.tv_nsec
3848         incl    hres_lock               / release the hres_lock
3849 
3850         popl    %ebx
3851         popl    %esi
3852         leave
3853         ret
3854         SET_SIZE(hres_tick)
3855 
3856 #endif  /* __i386 */
3857 #endif  /* __lint */
3858 
3859 /*
3860  * void prefetch_smap_w(void *)
3861  *
3862  * Prefetch ahead within a linear list of smap structures.
3863  * Not implemented for ia32.  Stub for compatibility.
3864  */
3865 
3866 #if defined(__lint)
3867 
3868 /*ARGSUSED*/
3869 void prefetch_smap_w(void *smp)
3870 {}
3871 
3872 #else   /* __lint */
3873 
3874         ENTRY(prefetch_smap_w)
3875         rep;    ret     /* use 2 byte return instruction when branch target */
3876                         /* AMD Software Optimization Guide - Section 6.2 */
3877         SET_SIZE(prefetch_smap_w)
3878 
3879 #endif  /* __lint */
3880 
3881 /*
3882  * prefetch_page_r(page_t *)
3883  * issue prefetch instructions for a page_t
3884  */
3885 #if defined(__lint)
3886 
3887 /*ARGSUSED*/
3888 void
3889 prefetch_page_r(void *pp)
3890 {}
3891 
3892 #else   /* __lint */
3893 
3894         ENTRY(prefetch_page_r)
3895         rep;    ret     /* use 2 byte return instruction when branch target */
3896                         /* AMD Software Optimization Guide - Section 6.2 */
3897         SET_SIZE(prefetch_page_r)
3898 
3899 #endif  /* __lint */
3900 
3901 #if defined(__lint)
3902 
3903 /*ARGSUSED*/
3904 int
3905 bcmp(const void *s1, const void *s2, size_t count)
3906 { return (0); }
3907 
3908 #else   /* __lint */
3909 
3910 #if defined(__amd64)
3911 
3912         ENTRY(bcmp)
3913         pushq   %rbp
3914         movq    %rsp, %rbp
3915 #ifdef DEBUG
3916         testq   %rdx,%rdx
3917         je      1f
3918         movq    postbootkernelbase(%rip), %r11
3919         cmpq    %r11, %rdi
3920         jb      0f
3921         cmpq    %r11, %rsi
3922         jnb     1f
3923 0:      leaq    .bcmp_panic_msg(%rip), %rdi
3924         xorl    %eax, %eax
3925         call    panic
3926 1:
3927 #endif  /* DEBUG */
3928         call    memcmp
3929         testl   %eax, %eax
3930         setne   %dl
3931         leave
3932         movzbl  %dl, %eax
3933         ret
3934         SET_SIZE(bcmp)
3935         
3936 #elif defined(__i386)
3937         
3938 #define ARG_S1          8
3939 #define ARG_S2          12
3940 #define ARG_LENGTH      16
3941 
3942         ENTRY(bcmp)
3943         pushl   %ebp
3944         movl    %esp, %ebp      / create new stack frame
3945 #ifdef DEBUG
3946         cmpl    $0, ARG_LENGTH(%ebp)
3947         je      1f
3948         movl    postbootkernelbase, %eax
3949         cmpl    %eax, ARG_S1(%ebp)
3950         jb      0f
3951         cmpl    %eax, ARG_S2(%ebp)
3952         jnb     1f
3953 0:      pushl   $.bcmp_panic_msg
3954         call    panic
3955 1:
3956 #endif  /* DEBUG */
3957 
3958         pushl   %edi            / save register variable
3959         movl    ARG_S1(%ebp), %eax      / %eax = address of string 1
3960         movl    ARG_S2(%ebp), %ecx      / %ecx = address of string 2
3961         cmpl    %eax, %ecx      / if the same string
3962         je      .equal          / goto .equal
3963         movl    ARG_LENGTH(%ebp), %edi  / %edi = length in bytes
3964         cmpl    $4, %edi        / if %edi < 4
3965         jb      .byte_check     / goto .byte_check
3966         .align  4
3967 .word_loop:
3968         movl    (%ecx), %edx    / move 1 word from (%ecx) to %edx
3969         leal    -4(%edi), %edi  / %edi -= 4
3970         cmpl    (%eax), %edx    / compare 1 word from (%eax) with %edx
3971         jne     .word_not_equal / if not equal, goto .word_not_equal
3972         leal    4(%ecx), %ecx   / %ecx += 4 (next word)
3973         leal    4(%eax), %eax   / %eax += 4 (next word)
3974         cmpl    $4, %edi        / if %edi >= 4
3975         jae     .word_loop      / goto .word_loop
3976 .byte_check:
3977         cmpl    $0, %edi        / if %edi == 0
3978         je      .equal          / goto .equal
3979         jmp     .byte_loop      / goto .byte_loop (checks in bytes)
3980 .word_not_equal:
3981         leal    4(%edi), %edi   / %edi += 4 (post-decremented)
3982         .align  4
3983 .byte_loop:
3984         movb    (%ecx), %dl     / move 1 byte from (%ecx) to %dl
3985         cmpb    %dl, (%eax)     / compare %dl with 1 byte from (%eax)
3986         jne     .not_equal      / if not equal, goto .not_equal
3987         incl    %ecx            / %ecx++ (next byte)
3988         incl    %eax            / %eax++ (next byte)
3989         decl    %edi            / %edi--
3990         jnz     .byte_loop      / if not zero, goto .byte_loop
3991 .equal:
3992         xorl    %eax, %eax      / %eax = 0
3993         popl    %edi            / restore register variable
3994         leave                   / restore old stack frame
3995         ret                     / return (NULL)
3996         .align  4
3997 .not_equal:
3998         movl    $1, %eax        / return 1
3999         popl    %edi            / restore register variable
4000         leave                   / restore old stack frame
4001         ret                     / return (NULL)
4002         SET_SIZE(bcmp)
4003 
4004 #endif  /* __i386 */
4005 
4006 #ifdef DEBUG
4007         .text
4008 .bcmp_panic_msg:
4009         .string "bcmp: arguments below kernelbase"
4010 #endif  /* DEBUG */
4011 
4012 #endif  /* __lint */
4013 
4014 #if defined(__lint)
4015 
4016 uint_t
4017 bsrw_insn(uint16_t mask)
4018 {
4019         uint_t index = sizeof (mask) * NBBY - 1;
4020 
4021         while ((mask & (1 << index)) == 0)
4022                 index--;
4023         return (index);
4024 }
4025 
4026 #else   /* __lint */
4027 
4028 #if defined(__amd64)
4029 
4030         ENTRY_NP(bsrw_insn)
4031         xorl    %eax, %eax
4032         bsrw    %di, %ax
4033         ret
4034         SET_SIZE(bsrw_insn)
4035 
4036 #elif defined(__i386)
4037 
4038         ENTRY_NP(bsrw_insn)
4039         movw    4(%esp), %cx
4040         xorl    %eax, %eax
4041         bsrw    %cx, %ax
4042         ret
4043         SET_SIZE(bsrw_insn)
4044 
4045 #endif  /* __i386 */
4046 #endif  /* __lint */
4047 
4048 #if defined(__lint)
4049 
4050 uint_t
4051 atomic_btr32(uint32_t *pending, uint_t pil)
4052 {
4053         return (*pending &= ~(1 << pil));
4054 }
4055 
4056 #else   /* __lint */
4057 
4058 #if defined(__i386)
4059 
4060         ENTRY_NP(atomic_btr32)
4061         movl    4(%esp), %ecx
4062         movl    8(%esp), %edx
4063         xorl    %eax, %eax
4064         lock
4065         btrl    %edx, (%ecx)
4066         setc    %al
4067         ret
4068         SET_SIZE(atomic_btr32)
4069 
4070 #endif  /* __i386 */
4071 #endif  /* __lint */
4072 
4073 #if defined(__lint)
4074 
4075 /*ARGSUSED*/
4076 void
4077 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4078             uint_t arg2)
4079 {}
4080 
4081 #else   /* __lint */
4082 
4083 #if defined(__amd64)
4084 
4085         ENTRY_NP(switch_sp_and_call)
4086         pushq   %rbp
4087         movq    %rsp, %rbp              /* set up stack frame */
4088         movq    %rdi, %rsp              /* switch stack pointer */
4089         movq    %rdx, %rdi              /* pass func arg 1 */
4090         movq    %rsi, %r11              /* save function to call */
4091         movq    %rcx, %rsi              /* pass func arg 2 */
4092         call    *%r11                   /* call function */
4093         leave                           /* restore stack */
4094         ret
4095         SET_SIZE(switch_sp_and_call)
4096 
4097 #elif defined(__i386)
4098 
4099         ENTRY_NP(switch_sp_and_call)
4100         pushl   %ebp
4101         mov     %esp, %ebp              /* set up stack frame */
4102         movl    8(%ebp), %esp           /* switch stack pointer */
4103         pushl   20(%ebp)                /* push func arg 2 */
4104         pushl   16(%ebp)                /* push func arg 1 */
4105         call    *12(%ebp)               /* call function */
4106         addl    $8, %esp                /* pop arguments */
4107         leave                           /* restore stack */
4108         ret
4109         SET_SIZE(switch_sp_and_call)
4110 
4111 #endif  /* __i386 */
4112 #endif  /* __lint */
4113 
4114 #if defined(__lint)
4115 
4116 void
4117 kmdb_enter(void)
4118 {}
4119 
4120 #else   /* __lint */
4121 
4122 #if defined(__amd64)
4123 
4124         ENTRY_NP(kmdb_enter)
4125         pushq   %rbp
4126         movq    %rsp, %rbp
4127 
4128         /*
4129          * Save flags, do a 'cli' then return the saved flags
4130          */
4131         call    intr_clear
4132 
4133         int     $T_DBGENTR
4134 
4135         /*
4136          * Restore the saved flags
4137          */
4138         movq    %rax, %rdi
4139         call    intr_restore
4140 
4141         leave
4142         ret     
4143         SET_SIZE(kmdb_enter)
4144 
4145 #elif defined(__i386)
4146 
4147         ENTRY_NP(kmdb_enter)
4148         pushl   %ebp
4149         movl    %esp, %ebp
4150 
4151         /*
4152          * Save flags, do a 'cli' then return the saved flags
4153          */
4154         call    intr_clear
4155 
4156         int     $T_DBGENTR
4157 
4158         /*
4159          * Restore the saved flags
4160          */
4161         pushl   %eax
4162         call    intr_restore
4163         addl    $4, %esp
4164 
4165         leave
4166         ret     
4167         SET_SIZE(kmdb_enter)
4168 
4169 #endif  /* __i386 */
4170 #endif  /* __lint */
4171 
4172 #if defined(__lint)
4173 
4174 void
4175 return_instr(void)
4176 {}
4177 
4178 #else   /* __lint */
4179 
4180         ENTRY_NP(return_instr)
4181         rep;    ret     /* use 2 byte instruction when branch target */
4182                         /* AMD Software Optimization Guide - Section 6.2 */
4183         SET_SIZE(return_instr)
4184 
4185 #endif  /* __lint */
4186 
4187 #if defined(__lint)
4188 
4189 ulong_t
4190 getflags(void)
4191 {
4192         return (0);
4193 }
4194 
4195 #else   /* __lint */
4196 
4197 #if defined(__amd64)
4198 
4199         ENTRY(getflags)
4200         pushfq
4201         popq    %rax
4202 #if defined(__xpv)
4203         CURTHREAD(%rdi)
4204         KPREEMPT_DISABLE(%rdi)
4205         /*
4206          * Synthesize the PS_IE bit from the event mask bit
4207          */
4208         CURVCPU(%r11)
4209         andq    $_BITNOT(PS_IE), %rax
4210         XEN_TEST_UPCALL_MASK(%r11)
4211         jnz     1f
4212         orq     $PS_IE, %rax
4213 1:
4214         KPREEMPT_ENABLE_NOKP(%rdi)
4215 #endif
4216         ret
4217         SET_SIZE(getflags)
4218 
4219 #elif defined(__i386)
4220 
4221         ENTRY(getflags)
4222         pushfl
4223         popl    %eax
4224 #if defined(__xpv)
4225         CURTHREAD(%ecx)
4226         KPREEMPT_DISABLE(%ecx)
4227         /*
4228          * Synthesize the PS_IE bit from the event mask bit
4229          */
4230         CURVCPU(%edx)
4231         andl    $_BITNOT(PS_IE), %eax
4232         XEN_TEST_UPCALL_MASK(%edx)
4233         jnz     1f
4234         orl     $PS_IE, %eax
4235 1:
4236         KPREEMPT_ENABLE_NOKP(%ecx)
4237 #endif
4238         ret
4239         SET_SIZE(getflags)
4240 
4241 #endif  /* __i386 */
4242 
4243 #endif  /* __lint */
4244 
4245 #if defined(__lint)
4246 
4247 ftrace_icookie_t
4248 ftrace_interrupt_disable(void)
4249 { return (0); }
4250 
4251 #else   /* __lint */
4252 
4253 #if defined(__amd64)
4254 
4255         ENTRY(ftrace_interrupt_disable)
4256         pushfq
4257         popq    %rax
4258         CLI(%rdx)
4259         ret
4260         SET_SIZE(ftrace_interrupt_disable)
4261 
4262 #elif defined(__i386)
4263                 
4264         ENTRY(ftrace_interrupt_disable)
4265         pushfl
4266         popl    %eax
4267         CLI(%edx)
4268         ret
4269         SET_SIZE(ftrace_interrupt_disable)
4270 
4271 #endif  /* __i386 */    
4272 #endif  /* __lint */
4273 
4274 #if defined(__lint)
4275 
4276 /*ARGSUSED*/
4277 void
4278 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4279 {}
4280 
4281 #else   /* __lint */
4282 
4283 #if defined(__amd64)
4284 
4285         ENTRY(ftrace_interrupt_enable)
4286         pushq   %rdi
4287         popfq
4288         ret
4289         SET_SIZE(ftrace_interrupt_enable)
4290 
4291 #elif defined(__i386)
4292                 
4293         ENTRY(ftrace_interrupt_enable)
4294         movl    4(%esp), %eax
4295         pushl   %eax
4296         popfl
4297         ret
4298         SET_SIZE(ftrace_interrupt_enable)
4299 
4300 #endif  /* __i386 */
4301 #endif  /* __lint */
4302 
4303 #if defined (__lint)
4304 
4305 /*ARGSUSED*/
4306 void
4307 clflush_insn(caddr_t addr)
4308 {}
4309 
4310 #else /* __lint */
4311 
4312 #if defined (__amd64)
4313         ENTRY(clflush_insn)
4314         clflush (%rdi)
4315         ret
4316         SET_SIZE(clflush_insn)
4317 #elif defined (__i386)
4318         ENTRY(clflush_insn)
4319         movl    4(%esp), %eax
4320         clflush (%eax)
4321         ret
4322         SET_SIZE(clflush_insn)
4323 
4324 #endif /* __i386 */
4325 #endif /* __lint */
4326 
4327 #if defined (__lint)
4328 /*ARGSUSED*/
4329 void
4330 mfence_insn(void)
4331 {}
4332 
4333 #else /* __lint */
4334 
4335 #if defined (__amd64)
4336         ENTRY(mfence_insn)
4337         mfence
4338         ret
4339         SET_SIZE(mfence_insn)
4340 #elif defined (__i386)
4341         ENTRY(mfence_insn)
4342         mfence
4343         ret
4344         SET_SIZE(mfence_insn)
4345 
4346 #endif /* __i386 */
4347 #endif /* __lint */
4348 
4349 /*
4350  * VMware implements an I/O port that programs can query to detect if software
4351  * is running in a VMware hypervisor. This hypervisor port behaves differently
4352  * depending on magic values in certain registers and modifies some registers
4353  * as a side effect.
4354  *
4355  * References: http://kb.vmware.com/kb/1009458 
4356  */
4357 
4358 #if defined(__lint)
4359 
4360 /* ARGSUSED */
4361 void
4362 vmware_port(int cmd, uint32_t *regs) { return; }
4363 
4364 #else
4365 
4366 #if defined(__amd64)
4367 
4368         ENTRY(vmware_port)
4369         pushq   %rbx
4370         movl    $VMWARE_HVMAGIC, %eax
4371         movl    $0xffffffff, %ebx
4372         movl    %edi, %ecx
4373         movl    $VMWARE_HVPORT, %edx
4374         inl     (%dx)
4375         movl    %eax, (%rsi)
4376         movl    %ebx, 4(%rsi)
4377         movl    %ecx, 8(%rsi)
4378         movl    %edx, 12(%rsi)
4379         popq    %rbx
4380         ret
4381         SET_SIZE(vmware_port)
4382 
4383 #elif defined(__i386)
4384 
4385         ENTRY(vmware_port)
4386         pushl   %ebx
4387         pushl   %esi
4388         movl    $VMWARE_HVMAGIC, %eax
4389         movl    $0xffffffff, %ebx
4390         movl    12(%esp), %ecx
4391         movl    $VMWARE_HVPORT, %edx
4392         inl     (%dx)
4393         movl    16(%esp), %esi
4394         movl    %eax, (%esi)
4395         movl    %ebx, 4(%esi)
4396         movl    %ecx, 8(%esi)
4397         movl    %edx, 12(%esi)
4398         popl    %esi
4399         popl    %ebx
4400         ret
4401         SET_SIZE(vmware_port)
4402 
4403 #endif /* __i386 */
4404 #endif /* __lint */
--- EOF ---