1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  25  * Copyright (c) 2014 by Delphix. All rights reserved.
  26  * Copyright 2016 Joyent, Inc.
  27  */
  28 
  29 /*
  30  *  Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
  31  *  Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
  32  *    All Rights Reserved
  33  */
  34 
  35 /*
  36  * Copyright (c) 2009, Intel Corporation.
  37  * All rights reserved.
  38  */
  39 
  40 /*
  41  * General assembly language routines.
  42  * It is the intent of this file to contain routines that are
  43  * independent of the specific kernel architecture, and those that are
  44  * common across kernel architectures.
  45  * As architectures diverge, and implementations of specific
  46  * architecture-dependent routines change, the routines should be moved
  47  * from this file into the respective ../`arch -k`/subr.s file.
  48  */
  49 
  50 #include <sys/asm_linkage.h>
  51 #include <sys/asm_misc.h>
  52 #include <sys/panic.h>
  53 #include <sys/ontrap.h>
  54 #include <sys/regset.h>
  55 #include <sys/privregs.h>
  56 #include <sys/reboot.h>
  57 #include <sys/psw.h>
  58 #include <sys/x86_archext.h>
  59 
  60 #if defined(__lint)
  61 #include <sys/types.h>
  62 #include <sys/systm.h>
  63 #include <sys/thread.h>
  64 #include <sys/archsystm.h>
  65 #include <sys/byteorder.h>
  66 #include <sys/dtrace.h>
  67 #include <sys/ftrace.h>
  68 #else   /* __lint */
  69 #include "assym.h"
  70 #endif  /* __lint */
  71 #include <sys/dditypes.h>
  72 
  73 /*
  74  * on_fault()
  75  *
  76  * Catch lofault faults. Like setjmp except it returns one
  77  * if code following causes uncorrectable fault. Turned off
  78  * by calling no_fault(). Note that while under on_fault(),
  79  * SMAP is disabled. For more information see
  80  * uts/intel/ia32/ml/copy.s.
  81  */
  82 
  83 #if defined(__lint)
  84 
  85 /* ARGSUSED */
  86 int
  87 on_fault(label_t *ljb)
  88 { return (0); }
  89 
  90 void
  91 no_fault(void)
  92 {}
  93 
  94 #else   /* __lint */
  95 
  96 #if defined(__amd64)
  97 
  98         ENTRY(on_fault)
  99         movq    %gs:CPU_THREAD, %rsi
 100         leaq    catch_fault(%rip), %rdx
 101         movq    %rdi, T_ONFAULT(%rsi)           /* jumpbuf in t_onfault */
 102         movq    %rdx, T_LOFAULT(%rsi)           /* catch_fault in t_lofault */
 103         call    smap_disable                    /* allow user accesses */
 104         jmp     setjmp                          /* let setjmp do the rest */
 105 
 106 catch_fault:
 107         movq    %gs:CPU_THREAD, %rsi
 108         movq    T_ONFAULT(%rsi), %rdi           /* address of save area */
 109         xorl    %eax, %eax
 110         movq    %rax, T_ONFAULT(%rsi)           /* turn off onfault */
 111         movq    %rax, T_LOFAULT(%rsi)           /* turn off lofault */
 112         call    smap_enable                     /* disallow user accesses */
 113         jmp     longjmp                         /* let longjmp do the rest */
 114         SET_SIZE(on_fault)
 115 
 116         ENTRY(no_fault)
 117         movq    %gs:CPU_THREAD, %rsi
 118         xorl    %eax, %eax
 119         movq    %rax, T_ONFAULT(%rsi)           /* turn off onfault */
 120         movq    %rax, T_LOFAULT(%rsi)           /* turn off lofault */
 121         call    smap_enable                     /* disallow user accesses */
 122         ret
 123         SET_SIZE(no_fault)
 124 
 125 #elif defined(__i386)
 126 
 127         ENTRY(on_fault)
 128         movl    %gs:CPU_THREAD, %edx
 129         movl    4(%esp), %eax                   /* jumpbuf address */
 130         leal    catch_fault, %ecx
 131         movl    %eax, T_ONFAULT(%edx)           /* jumpbuf in t_onfault */
 132         movl    %ecx, T_LOFAULT(%edx)           /* catch_fault in t_lofault */
 133         jmp     setjmp                          /* let setjmp do the rest */
 134 
 135 catch_fault:
 136         movl    %gs:CPU_THREAD, %edx
 137         xorl    %eax, %eax
 138         movl    T_ONFAULT(%edx), %ecx           /* address of save area */
 139         movl    %eax, T_ONFAULT(%edx)           /* turn off onfault */
 140         movl    %eax, T_LOFAULT(%edx)           /* turn off lofault */
 141         pushl   %ecx
 142         call    longjmp                         /* let longjmp do the rest */
 143         SET_SIZE(on_fault)
 144 
 145         ENTRY(no_fault)
 146         movl    %gs:CPU_THREAD, %edx
 147         xorl    %eax, %eax
 148         movl    %eax, T_ONFAULT(%edx)           /* turn off onfault */
 149         movl    %eax, T_LOFAULT(%edx)           /* turn off lofault */
 150         ret
 151         SET_SIZE(no_fault)
 152 
 153 #endif  /* __i386 */
 154 #endif  /* __lint */
 155 
 156 /*
 157  * Default trampoline code for on_trap() (see <sys/ontrap.h>).  We just
 158  * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
 159  */
 160 
 161 #if defined(lint)
 162 
 163 void
 164 on_trap_trampoline(void)
 165 {}
 166 
 167 #else   /* __lint */
 168 
 169 #if defined(__amd64)
 170 
 171         ENTRY(on_trap_trampoline)
 172         movq    %gs:CPU_THREAD, %rsi
 173         movq    T_ONTRAP(%rsi), %rdi
 174         addq    $OT_JMPBUF, %rdi
 175         jmp     longjmp
 176         SET_SIZE(on_trap_trampoline)
 177 
 178 #elif defined(__i386)
 179 
 180         ENTRY(on_trap_trampoline)
 181         movl    %gs:CPU_THREAD, %eax
 182         movl    T_ONTRAP(%eax), %eax
 183         addl    $OT_JMPBUF, %eax
 184         pushl   %eax
 185         call    longjmp
 186         SET_SIZE(on_trap_trampoline)
 187 
 188 #endif  /* __i386 */
 189 #endif  /* __lint */
 190 
 191 /*
 192  * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
 193  * more information about the on_trap() mechanism.  If the on_trap_data is the
 194  * same as the topmost stack element, we just modify that element.
 195  */
 196 #if defined(lint)
 197 
 198 /*ARGSUSED*/
 199 int
 200 on_trap(on_trap_data_t *otp, uint_t prot)
 201 { return (0); }
 202 
 203 #else   /* __lint */
 204 
 205 #if defined(__amd64)
 206 
 207         ENTRY(on_trap)
 208         movw    %si, OT_PROT(%rdi)              /* ot_prot = prot */
 209         movw    $0, OT_TRAP(%rdi)               /* ot_trap = 0 */
 210         leaq    on_trap_trampoline(%rip), %rdx  /* rdx = &on_trap_trampoline */
 211         movq    %rdx, OT_TRAMPOLINE(%rdi)       /* ot_trampoline = rdx */
 212         xorl    %ecx, %ecx
 213         movq    %rcx, OT_HANDLE(%rdi)           /* ot_handle = NULL */
 214         movq    %rcx, OT_PAD1(%rdi)             /* ot_pad1 = NULL */
 215         movq    %gs:CPU_THREAD, %rdx            /* rdx = curthread */
 216         movq    T_ONTRAP(%rdx), %rcx            /* rcx = curthread->t_ontrap */
 217         cmpq    %rdi, %rcx                      /* if (otp == %rcx)     */
 218         je      0f                              /*      don't modify t_ontrap */
 219 
 220         movq    %rcx, OT_PREV(%rdi)             /* ot_prev = t_ontrap */
 221         movq    %rdi, T_ONTRAP(%rdx)            /* curthread->t_ontrap = otp */
 222 
 223 0:      addq    $OT_JMPBUF, %rdi                /* &ot_jmpbuf */
 224         jmp     setjmp
 225         SET_SIZE(on_trap)
 226 
 227 #elif defined(__i386)
 228 
 229         ENTRY(on_trap)
 230         movl    4(%esp), %eax                   /* %eax = otp */
 231         movl    8(%esp), %edx                   /* %edx = prot */
 232 
 233         movw    %dx, OT_PROT(%eax)              /* ot_prot = prot */
 234         movw    $0, OT_TRAP(%eax)               /* ot_trap = 0 */
 235         leal    on_trap_trampoline, %edx        /* %edx = &on_trap_trampoline */
 236         movl    %edx, OT_TRAMPOLINE(%eax)       /* ot_trampoline = %edx */
 237         movl    $0, OT_HANDLE(%eax)             /* ot_handle = NULL */
 238         movl    $0, OT_PAD1(%eax)               /* ot_pad1 = NULL */
 239         movl    %gs:CPU_THREAD, %edx            /* %edx = curthread */
 240         movl    T_ONTRAP(%edx), %ecx            /* %ecx = curthread->t_ontrap */
 241         cmpl    %eax, %ecx                      /* if (otp == %ecx) */
 242         je      0f                              /*    don't modify t_ontrap */
 243 
 244         movl    %ecx, OT_PREV(%eax)             /* ot_prev = t_ontrap */
 245         movl    %eax, T_ONTRAP(%edx)            /* curthread->t_ontrap = otp */
 246 
 247 0:      addl    $OT_JMPBUF, %eax                /* %eax = &ot_jmpbuf */
 248         movl    %eax, 4(%esp)                   /* put %eax back on the stack */
 249         jmp     setjmp                          /* let setjmp do the rest */
 250         SET_SIZE(on_trap)
 251 
 252 #endif  /* __i386 */
 253 #endif  /* __lint */
 254 
 255 /*
 256  * Setjmp and longjmp implement non-local gotos using state vectors
 257  * type label_t.
 258  */
 259 
 260 #if defined(__lint)
 261 
 262 /* ARGSUSED */
 263 int
 264 setjmp(label_t *lp)
 265 { return (0); }
 266 
 267 /* ARGSUSED */
 268 void
 269 longjmp(label_t *lp)
 270 {}
 271 
 272 #else   /* __lint */
 273 
 274 #if LABEL_PC != 0
 275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
 276 #endif  /* LABEL_PC != 0 */
 277 
 278 #if defined(__amd64)
 279 
 280         ENTRY(setjmp)
 281         movq    %rsp, LABEL_SP(%rdi)
 282         movq    %rbp, LABEL_RBP(%rdi)
 283         movq    %rbx, LABEL_RBX(%rdi)
 284         movq    %r12, LABEL_R12(%rdi)
 285         movq    %r13, LABEL_R13(%rdi)
 286         movq    %r14, LABEL_R14(%rdi)
 287         movq    %r15, LABEL_R15(%rdi)
 288         movq    (%rsp), %rdx            /* return address */
 289         movq    %rdx, (%rdi)            /* LABEL_PC is 0 */
 290         xorl    %eax, %eax              /* return 0 */
 291         ret
 292         SET_SIZE(setjmp)
 293 
 294         ENTRY(longjmp)
 295         movq    LABEL_SP(%rdi), %rsp
 296         movq    LABEL_RBP(%rdi), %rbp
 297         movq    LABEL_RBX(%rdi), %rbx
 298         movq    LABEL_R12(%rdi), %r12
 299         movq    LABEL_R13(%rdi), %r13
 300         movq    LABEL_R14(%rdi), %r14
 301         movq    LABEL_R15(%rdi), %r15
 302         movq    (%rdi), %rdx            /* return address; LABEL_PC is 0 */
 303         movq    %rdx, (%rsp)
 304         xorl    %eax, %eax
 305         incl    %eax                    /* return 1 */
 306         ret
 307         SET_SIZE(longjmp)
 308 
 309 #elif defined(__i386)
 310 
 311         ENTRY(setjmp)
 312         movl    4(%esp), %edx           /* address of save area */
 313         movl    %ebp, LABEL_EBP(%edx)
 314         movl    %ebx, LABEL_EBX(%edx)
 315         movl    %esi, LABEL_ESI(%edx)
 316         movl    %edi, LABEL_EDI(%edx)
 317         movl    %esp, 4(%edx)
 318         movl    (%esp), %ecx            /* %eip (return address) */
 319         movl    %ecx, (%edx)            /* LABEL_PC is 0 */
 320         subl    %eax, %eax              /* return 0 */
 321         ret
 322         SET_SIZE(setjmp)
 323 
 324         ENTRY(longjmp)
 325         movl    4(%esp), %edx           /* address of save area */
 326         movl    LABEL_EBP(%edx), %ebp
 327         movl    LABEL_EBX(%edx), %ebx
 328         movl    LABEL_ESI(%edx), %esi
 329         movl    LABEL_EDI(%edx), %edi
 330         movl    4(%edx), %esp
 331         movl    (%edx), %ecx            /* %eip (return addr); LABEL_PC is 0 */
 332         movl    $1, %eax
 333         addl    $4, %esp                /* pop ret adr */
 334         jmp     *%ecx                   /* indirect */
 335         SET_SIZE(longjmp)
 336 
 337 #endif  /* __i386 */
 338 #endif  /* __lint */
 339 
 340 /*
 341  * if a() calls b() calls caller(),
 342  * caller() returns return address in a().
 343  * (Note: We assume a() and b() are C routines which do the normal entry/exit
 344  *  sequence.)
 345  */
 346 
 347 #if defined(__lint)
 348 
 349 caddr_t
 350 caller(void)
 351 { return (0); }
 352 
 353 #else   /* __lint */
 354 
 355 #if defined(__amd64)
 356 
 357         ENTRY(caller)
 358         movq    8(%rbp), %rax           /* b()'s return pc, in a() */
 359         ret
 360         SET_SIZE(caller)
 361 
 362 #elif defined(__i386)
 363 
 364         ENTRY(caller)
 365         movl    4(%ebp), %eax           /* b()'s return pc, in a() */
 366         ret
 367         SET_SIZE(caller)
 368 
 369 #endif  /* __i386 */
 370 #endif  /* __lint */
 371 
 372 /*
 373  * if a() calls callee(), callee() returns the
 374  * return address in a();
 375  */
 376 
 377 #if defined(__lint)
 378 
 379 caddr_t
 380 callee(void)
 381 { return (0); }
 382 
 383 #else   /* __lint */
 384 
 385 #if defined(__amd64)
 386 
 387         ENTRY(callee)
 388         movq    (%rsp), %rax            /* callee()'s return pc, in a() */
 389         ret
 390         SET_SIZE(callee)
 391 
 392 #elif defined(__i386)
 393 
 394         ENTRY(callee)
 395         movl    (%esp), %eax            /* callee()'s return pc, in a() */
 396         ret
 397         SET_SIZE(callee)
 398 
 399 #endif  /* __i386 */
 400 #endif  /* __lint */
 401 
 402 /*
 403  * return the current frame pointer
 404  */
 405 
 406 #if defined(__lint)
 407 
 408 greg_t
 409 getfp(void)
 410 { return (0); }
 411 
 412 #else   /* __lint */
 413 
 414 #if defined(__amd64)
 415 
 416         ENTRY(getfp)
 417         movq    %rbp, %rax
 418         ret
 419         SET_SIZE(getfp)
 420 
 421 #elif defined(__i386)
 422 
 423         ENTRY(getfp)
 424         movl    %ebp, %eax
 425         ret
 426         SET_SIZE(getfp)
 427 
 428 #endif  /* __i386 */
 429 #endif  /* __lint */
 430 
 431 /*
 432  * Invalidate a single page table entry in the TLB
 433  */
 434 
 435 #if defined(__lint)
 436 
 437 /* ARGSUSED */
 438 void
 439 mmu_tlbflush_entry(caddr_t m)
 440 {}
 441 
 442 #else   /* __lint */
 443 
 444 #if defined(__amd64)
 445 
 446         ENTRY(mmu_tlbflush_entry)
 447         invlpg  (%rdi)
 448         ret
 449         SET_SIZE(mmu_tlbflush_entry)
 450 
 451 #elif defined(__i386)
 452 
 453         ENTRY(mmu_tlbflush_entry)
 454         movl    4(%esp), %eax
 455         invlpg  (%eax)
 456         ret
 457         SET_SIZE(mmu_tlbflush_entry)
 458 
 459 #endif  /* __i386 */
 460 #endif  /* __lint */
 461 
 462 
 463 /*
 464  * Get/Set the value of various control registers
 465  */
 466 
 467 #if defined(__lint)
 468 
 469 ulong_t
 470 getcr0(void)
 471 { return (0); }
 472 
 473 /* ARGSUSED */
 474 void
 475 setcr0(ulong_t value)
 476 {}
 477 
 478 ulong_t
 479 getcr2(void)
 480 { return (0); }
 481 
 482 ulong_t
 483 getcr3(void)
 484 { return (0); }
 485 
 486 #if !defined(__xpv)
 487 /* ARGSUSED */
 488 void
 489 setcr3(ulong_t val)
 490 {}
 491 
 492 void
 493 reload_cr3(void)
 494 {}
 495 #endif
 496 
 497 ulong_t
 498 getcr4(void)
 499 { return (0); }
 500 
 501 /* ARGSUSED */
 502 void
 503 setcr4(ulong_t val)
 504 {}
 505 
 506 #if defined(__amd64)
 507 
 508 ulong_t
 509 getcr8(void)
 510 { return (0); }
 511 
 512 /* ARGSUSED */
 513 void
 514 setcr8(ulong_t val)
 515 {}
 516 
 517 #endif  /* __amd64 */
 518 
 519 #else   /* __lint */
 520 
 521 #if defined(__amd64)
 522 
 523         ENTRY(getcr0)
 524         movq    %cr0, %rax
 525         ret
 526         SET_SIZE(getcr0)
 527 
 528         ENTRY(setcr0)
 529         movq    %rdi, %cr0
 530         ret
 531         SET_SIZE(setcr0)
 532 
 533         ENTRY(getcr2)
 534 #if defined(__xpv)
 535         movq    %gs:CPU_VCPU_INFO, %rax
 536         movq    VCPU_INFO_ARCH_CR2(%rax), %rax
 537 #else
 538         movq    %cr2, %rax
 539 #endif
 540         ret
 541         SET_SIZE(getcr2)
 542 
 543         ENTRY(getcr3)
 544         movq    %cr3, %rax
 545         ret
 546         SET_SIZE(getcr3)
 547 
 548 #if !defined(__xpv)
 549 
 550         ENTRY(setcr3)
 551         movq    %rdi, %cr3
 552         ret
 553         SET_SIZE(setcr3)
 554 
 555         ENTRY(reload_cr3)
 556         movq    %cr3, %rdi
 557         movq    %rdi, %cr3
 558         ret
 559         SET_SIZE(reload_cr3)
 560 
 561 #endif  /* __xpv */
 562 
 563         ENTRY(getcr4)
 564         movq    %cr4, %rax
 565         ret
 566         SET_SIZE(getcr4)
 567 
 568         ENTRY(setcr4)
 569         movq    %rdi, %cr4
 570         ret
 571         SET_SIZE(setcr4)
 572 
 573         ENTRY(getcr8)
 574         movq    %cr8, %rax
 575         ret
 576         SET_SIZE(getcr8)
 577 
 578         ENTRY(setcr8)
 579         movq    %rdi, %cr8
 580         ret
 581         SET_SIZE(setcr8)
 582 
 583 #elif defined(__i386)
 584 
 585         ENTRY(getcr0)
 586         movl    %cr0, %eax
 587         ret
 588         SET_SIZE(getcr0)
 589 
 590         ENTRY(setcr0)
 591         movl    4(%esp), %eax
 592         movl    %eax, %cr0
 593         ret
 594         SET_SIZE(setcr0)
 595 
 596         /*
 597          * "lock mov %cr0" is used on processors which indicate it is
 598          * supported via CPUID. Normally the 32 bit TPR is accessed via
 599          * the local APIC.
 600          */
 601         ENTRY(getcr8)
 602         lock
 603         movl    %cr0, %eax
 604         ret
 605         SET_SIZE(getcr8)
 606 
 607         ENTRY(setcr8)
 608         movl    4(%esp), %eax
 609         lock
 610         movl    %eax, %cr0
 611         ret
 612         SET_SIZE(setcr8)
 613 
 614         ENTRY(getcr2)
 615 #if defined(__xpv)
 616         movl    %gs:CPU_VCPU_INFO, %eax
 617         movl    VCPU_INFO_ARCH_CR2(%eax), %eax
 618 #else
 619         movl    %cr2, %eax
 620 #endif
 621         ret
 622         SET_SIZE(getcr2)
 623 
 624         ENTRY(getcr3)
 625         movl    %cr3, %eax
 626         ret
 627         SET_SIZE(getcr3)
 628 
 629 #if !defined(__xpv)
 630 
 631         ENTRY(setcr3)
 632         movl    4(%esp), %eax
 633         movl    %eax, %cr3
 634         ret
 635         SET_SIZE(setcr3)
 636 
 637         ENTRY(reload_cr3)
 638         movl    %cr3, %eax
 639         movl    %eax, %cr3
 640         ret
 641         SET_SIZE(reload_cr3)
 642 
 643 #endif  /* __xpv */
 644 
 645         ENTRY(getcr4)
 646         movl    %cr4, %eax
 647         ret
 648         SET_SIZE(getcr4)
 649 
 650         ENTRY(setcr4)
 651         movl    4(%esp), %eax
 652         movl    %eax, %cr4
 653         ret
 654         SET_SIZE(setcr4)
 655 
 656 #endif  /* __i386 */
 657 #endif  /* __lint */
 658 
 659 #if defined(__lint)
 660 
 661 /*ARGSUSED*/
 662 uint32_t
 663 __cpuid_insn(struct cpuid_regs *regs)
 664 { return (0); }
 665 
 666 #else   /* __lint */
 667 
 668 #if defined(__amd64)
 669 
 670         ENTRY(__cpuid_insn)
 671         movq    %rbx, %r8
 672         movq    %rcx, %r9
 673         movq    %rdx, %r11
 674         movl    (%rdi), %eax            /* %eax = regs->cp_eax */
 675         movl    0x4(%rdi), %ebx         /* %ebx = regs->cp_ebx */
 676         movl    0x8(%rdi), %ecx         /* %ecx = regs->cp_ecx */
 677         movl    0xc(%rdi), %edx         /* %edx = regs->cp_edx */
 678         cpuid
 679         movl    %eax, (%rdi)            /* regs->cp_eax = %eax */
 680         movl    %ebx, 0x4(%rdi)         /* regs->cp_ebx = %ebx */
 681         movl    %ecx, 0x8(%rdi)         /* regs->cp_ecx = %ecx */
 682         movl    %edx, 0xc(%rdi)         /* regs->cp_edx = %edx */
 683         movq    %r8, %rbx
 684         movq    %r9, %rcx
 685         movq    %r11, %rdx
 686         ret
 687         SET_SIZE(__cpuid_insn)
 688 
 689 #elif defined(__i386)
 690 
 691         ENTRY(__cpuid_insn)
 692         pushl   %ebp
 693         movl    0x8(%esp), %ebp         /* %ebp = regs */
 694         pushl   %ebx
 695         pushl   %ecx
 696         pushl   %edx
 697         movl    (%ebp), %eax            /* %eax = regs->cp_eax */
 698         movl    0x4(%ebp), %ebx         /* %ebx = regs->cp_ebx */
 699         movl    0x8(%ebp), %ecx         /* %ecx = regs->cp_ecx */
 700         movl    0xc(%ebp), %edx         /* %edx = regs->cp_edx */
 701         cpuid
 702         movl    %eax, (%ebp)            /* regs->cp_eax = %eax */
 703         movl    %ebx, 0x4(%ebp)         /* regs->cp_ebx = %ebx */
 704         movl    %ecx, 0x8(%ebp)         /* regs->cp_ecx = %ecx */
 705         movl    %edx, 0xc(%ebp)         /* regs->cp_edx = %edx */
 706         popl    %edx
 707         popl    %ecx
 708         popl    %ebx
 709         popl    %ebp
 710         ret
 711         SET_SIZE(__cpuid_insn)
 712 
 713 #endif  /* __i386 */
 714 #endif  /* __lint */
 715 
 716 #if defined(__lint)
 717 
 718 /*ARGSUSED*/
 719 void
 720 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
 721 {}
 722 
 723 #else   /* __lint */
 724 
 725 #if defined(__amd64)
 726 
 727         ENTRY_NP(i86_monitor)
 728         pushq   %rbp
 729         movq    %rsp, %rbp
 730         movq    %rdi, %rax              /* addr */
 731         movq    %rsi, %rcx              /* extensions */
 732         /* rdx contains input arg3: hints */
 733         clflush (%rax)
 734         .byte   0x0f, 0x01, 0xc8        /* monitor */
 735         leave
 736         ret
 737         SET_SIZE(i86_monitor)
 738 
 739 #elif defined(__i386)
 740 
 741 ENTRY_NP(i86_monitor)
 742         pushl   %ebp
 743         movl    %esp, %ebp
 744         movl    0x8(%ebp),%eax          /* addr */
 745         movl    0xc(%ebp),%ecx          /* extensions */
 746         movl    0x10(%ebp),%edx         /* hints */
 747         clflush (%eax)
 748         .byte   0x0f, 0x01, 0xc8        /* monitor */
 749         leave
 750         ret
 751         SET_SIZE(i86_monitor)
 752 
 753 #endif  /* __i386 */
 754 #endif  /* __lint */
 755 
 756 #if defined(__lint)
 757 
 758 /*ARGSUSED*/
 759 void
 760 i86_mwait(uint32_t data, uint32_t extensions)
 761 {}
 762 
 763 #else   /* __lint */
 764 
 765 #if defined(__amd64)
 766 
 767         ENTRY_NP(i86_mwait)
 768         pushq   %rbp
 769         movq    %rsp, %rbp
 770         movq    %rdi, %rax              /* data */
 771         movq    %rsi, %rcx              /* extensions */
 772         .byte   0x0f, 0x01, 0xc9        /* mwait */
 773         leave
 774         ret
 775         SET_SIZE(i86_mwait)
 776 
 777 #elif defined(__i386)
 778 
 779         ENTRY_NP(i86_mwait)
 780         pushl   %ebp
 781         movl    %esp, %ebp
 782         movl    0x8(%ebp),%eax          /* data */
 783         movl    0xc(%ebp),%ecx          /* extensions */
 784         .byte   0x0f, 0x01, 0xc9        /* mwait */
 785         leave
 786         ret
 787         SET_SIZE(i86_mwait)
 788 
 789 #endif  /* __i386 */
 790 #endif  /* __lint */
 791 
 792 #if defined(__xpv)
 793         /*
 794          * Defined in C
 795          */
 796 #else
 797 
 798 #if defined(__lint)
 799 
 800 hrtime_t
 801 tsc_read(void)
 802 {
 803         return (0);
 804 }
 805 
 806 #else   /* __lint */
 807 
 808 #if defined(__amd64)
 809 
 810         ENTRY_NP(tsc_read)
 811         movq    %rbx, %r11
 812         movl    $0, %eax
 813         cpuid
 814         rdtsc
 815         movq    %r11, %rbx
 816         shlq    $32, %rdx
 817         orq     %rdx, %rax
 818         ret
 819         .globl _tsc_mfence_start
 820 _tsc_mfence_start:
 821         mfence
 822         rdtsc
 823         shlq    $32, %rdx
 824         orq     %rdx, %rax
 825         ret
 826         .globl _tsc_mfence_end
 827 _tsc_mfence_end:
 828         .globl _tscp_start
 829 _tscp_start:
 830         .byte   0x0f, 0x01, 0xf9        /* rdtscp instruction */
 831         shlq    $32, %rdx
 832         orq     %rdx, %rax
 833         ret
 834         .globl _tscp_end
 835 _tscp_end:
 836         .globl _no_rdtsc_start
 837 _no_rdtsc_start:
 838         xorl    %edx, %edx
 839         xorl    %eax, %eax
 840         ret
 841         .globl _no_rdtsc_end
 842 _no_rdtsc_end:
 843         .globl _tsc_lfence_start
 844 _tsc_lfence_start:
 845         lfence
 846         rdtsc
 847         shlq    $32, %rdx
 848         orq     %rdx, %rax
 849         ret
 850         .globl _tsc_lfence_end
 851 _tsc_lfence_end:
 852         SET_SIZE(tsc_read)
 853 
 854 #else /* __i386 */
 855 
 856         ENTRY_NP(tsc_read)
 857         pushl   %ebx
 858         movl    $0, %eax
 859         cpuid
 860         rdtsc
 861         popl    %ebx
 862         ret
 863         .globl _tsc_mfence_start
 864 _tsc_mfence_start:
 865         mfence
 866         rdtsc
 867         ret
 868         .globl _tsc_mfence_end
 869 _tsc_mfence_end:
 870         .globl  _tscp_start
 871 _tscp_start:
 872         .byte   0x0f, 0x01, 0xf9        /* rdtscp instruction */
 873         ret
 874         .globl _tscp_end
 875 _tscp_end:
 876         .globl _no_rdtsc_start
 877 _no_rdtsc_start:
 878         xorl    %edx, %edx
 879         xorl    %eax, %eax
 880         ret
 881         .globl _no_rdtsc_end
 882 _no_rdtsc_end:
 883         .globl _tsc_lfence_start
 884 _tsc_lfence_start:
 885         lfence
 886         rdtsc
 887         ret
 888         .globl _tsc_lfence_end
 889 _tsc_lfence_end:
 890         SET_SIZE(tsc_read)
 891 
 892 #endif  /* __i386 */
 893 
 894 #endif  /* __lint */
 895 
 896 
 897 #endif  /* __xpv */
 898 
 899 #ifdef __lint
 900 /*
 901  * Do not use this function for obtaining clock tick.  This
 902  * is called by callers who do not need to have a guarenteed
 903  * correct tick value.  The proper routine to use is tsc_read().
 904  */
 905 u_longlong_t
 906 randtick(void)
 907 {
 908         return (0);
 909 }
 910 #else
 911 #if defined(__amd64)
 912         ENTRY_NP(randtick)
 913         rdtsc
 914         shlq    $32, %rdx
 915         orq     %rdx, %rax
 916         ret
 917         SET_SIZE(randtick)
 918 #else
 919         ENTRY_NP(randtick)
 920         rdtsc
 921         ret
 922         SET_SIZE(randtick)
 923 #endif /* __i386 */
 924 #endif /* __lint */
 925 /*
 926  * Insert entryp after predp in a doubly linked list.
 927  */
 928 
 929 #if defined(__lint)
 930 
 931 /*ARGSUSED*/
 932 void
 933 _insque(caddr_t entryp, caddr_t predp)
 934 {}
 935 
 936 #else   /* __lint */
 937 
 938 #if defined(__amd64)
 939 
 940         ENTRY(_insque)
 941         movq    (%rsi), %rax            /* predp->forw                       */
 942         movq    %rsi, CPTRSIZE(%rdi)    /* entryp->back = predp              */
 943         movq    %rax, (%rdi)            /* entryp->forw = predp->forw     */
 944         movq    %rdi, (%rsi)            /* predp->forw = entryp              */
 945         movq    %rdi, CPTRSIZE(%rax)    /* predp->forw->back = entryp     */
 946         ret
 947         SET_SIZE(_insque)
 948 
 949 #elif defined(__i386)
 950 
 951         ENTRY(_insque)
 952         movl    8(%esp), %edx
 953         movl    4(%esp), %ecx
 954         movl    (%edx), %eax            /* predp->forw                       */
 955         movl    %edx, CPTRSIZE(%ecx)    /* entryp->back = predp              */
 956         movl    %eax, (%ecx)            /* entryp->forw = predp->forw     */
 957         movl    %ecx, (%edx)            /* predp->forw = entryp              */
 958         movl    %ecx, CPTRSIZE(%eax)    /* predp->forw->back = entryp     */
 959         ret
 960         SET_SIZE(_insque)
 961 
 962 #endif  /* __i386 */
 963 #endif  /* __lint */
 964 
 965 /*
 966  * Remove entryp from a doubly linked list
 967  */
 968 
 969 #if defined(__lint)
 970 
 971 /*ARGSUSED*/
 972 void
 973 _remque(caddr_t entryp)
 974 {}
 975 
 976 #else   /* __lint */
 977 
 978 #if defined(__amd64)
 979 
 980         ENTRY(_remque)
 981         movq    (%rdi), %rax            /* entry->forw */
 982         movq    CPTRSIZE(%rdi), %rdx    /* entry->back */
 983         movq    %rax, (%rdx)            /* entry->back->forw = entry->forw */
 984         movq    %rdx, CPTRSIZE(%rax)    /* entry->forw->back = entry->back */
 985         ret
 986         SET_SIZE(_remque)
 987 
 988 #elif defined(__i386)
 989 
 990         ENTRY(_remque)
 991         movl    4(%esp), %ecx
 992         movl    (%ecx), %eax            /* entry->forw */
 993         movl    CPTRSIZE(%ecx), %edx    /* entry->back */
 994         movl    %eax, (%edx)            /* entry->back->forw = entry->forw */
 995         movl    %edx, CPTRSIZE(%eax)    /* entry->forw->back = entry->back */
 996         ret
 997         SET_SIZE(_remque)
 998 
 999 #endif  /* __i386 */
1000 #endif  /* __lint */
1001 
1002 /*
1003  * Returns the number of
1004  * non-NULL bytes in string argument.
1005  */
1006 
1007 #if defined(__lint)
1008 
1009 /* ARGSUSED */
1010 size_t
1011 strlen(const char *str)
1012 { return (0); }
1013 
1014 #else   /* __lint */
1015 
1016 #if defined(__amd64)
1017 
1018 /*
1019  * This is close to a simple transliteration of a C version of this
1020  * routine.  We should either just -make- this be a C version, or
1021  * justify having it in assembler by making it significantly faster.
1022  *
1023  * size_t
1024  * strlen(const char *s)
1025  * {
1026  *      const char *s0;
1027  * #if defined(DEBUG)
1028  *      if ((uintptr_t)s < KERNELBASE)
1029  *              panic(.str_panic_msg);
1030  * #endif
1031  *      for (s0 = s; *s; s++)
1032  *              ;
1033  *      return (s - s0);
1034  * }
1035  */
1036 
1037         ENTRY(strlen)
1038 #ifdef DEBUG
1039         movq    postbootkernelbase(%rip), %rax
1040         cmpq    %rax, %rdi
1041         jae     str_valid
1042         pushq   %rbp
1043         movq    %rsp, %rbp
1044         leaq    .str_panic_msg(%rip), %rdi
1045         xorl    %eax, %eax
1046         call    panic
1047 #endif  /* DEBUG */
1048 str_valid:
1049         cmpb    $0, (%rdi)
1050         movq    %rdi, %rax
1051         je      .null_found
1052         .align  4
1053 .strlen_loop:
1054         incq    %rdi
1055         cmpb    $0, (%rdi)
1056         jne     .strlen_loop
1057 .null_found:
1058         subq    %rax, %rdi
1059         movq    %rdi, %rax
1060         ret
1061         SET_SIZE(strlen)
1062 
1063 #elif defined(__i386)
1064 
1065         ENTRY(strlen)
1066 #ifdef DEBUG
1067         movl    postbootkernelbase, %eax
1068         cmpl    %eax, 4(%esp)
1069         jae     str_valid
1070         pushl   %ebp
1071         movl    %esp, %ebp
1072         pushl   $.str_panic_msg
1073         call    panic
1074 #endif /* DEBUG */
1075 
1076 str_valid:
1077         movl    4(%esp), %eax           /* %eax = string address */
1078         testl   $3, %eax                /* if %eax not word aligned */
1079         jnz     .not_word_aligned       /* goto .not_word_aligned */
1080         .align  4
1081 .word_aligned:
1082         movl    (%eax), %edx            /* move 1 word from (%eax) to %edx */
1083         movl    $0x7f7f7f7f, %ecx
1084         andl    %edx, %ecx              /* %ecx = %edx & 0x7f7f7f7f */
1085         addl    $4, %eax                /* next word */
1086         addl    $0x7f7f7f7f, %ecx       /* %ecx += 0x7f7f7f7f */
1087         orl     %edx, %ecx              /* %ecx |= %edx */
1088         andl    $0x80808080, %ecx       /* %ecx &= 0x80808080 */
1089         cmpl    $0x80808080, %ecx       /* if no null byte in this word */
1090         je      .word_aligned           /* goto .word_aligned */
1091         subl    $4, %eax                /* post-incremented */
1092 .not_word_aligned:
1093         cmpb    $0, (%eax)              /* if a byte in (%eax) is null */
1094         je      .null_found             /* goto .null_found */
1095         incl    %eax                    /* next byte */
1096         testl   $3, %eax                /* if %eax not word aligned */
1097         jnz     .not_word_aligned       /* goto .not_word_aligned */
1098         jmp     .word_aligned           /* goto .word_aligned */
1099         .align  4
1100 .null_found:
1101         subl    4(%esp), %eax           /* %eax -= string address */
1102         ret
1103         SET_SIZE(strlen)
1104 
1105 #endif  /* __i386 */
1106 
1107 #ifdef DEBUG
1108         .text
1109 .str_panic_msg:
1110         .string "strlen: argument below kernelbase"
1111 #endif /* DEBUG */
1112 
1113 #endif  /* __lint */
1114 
1115         /*
1116          * Berkeley 4.3 introduced symbolically named interrupt levels
1117          * as a way deal with priority in a machine independent fashion.
1118          * Numbered priorities are machine specific, and should be
1119          * discouraged where possible.
1120          *
1121          * Note, for the machine specific priorities there are
1122          * examples listed for devices that use a particular priority.
1123          * It should not be construed that all devices of that
1124          * type should be at that priority.  It is currently were
1125          * the current devices fit into the priority scheme based
1126          * upon time criticalness.
1127          *
1128          * The underlying assumption of these assignments is that
1129          * IPL 10 is the highest level from which a device
1130          * routine can call wakeup.  Devices that interrupt from higher
1131          * levels are restricted in what they can do.  If they need
1132          * kernels services they should schedule a routine at a lower
1133          * level (via software interrupt) to do the required
1134          * processing.
1135          *
1136          * Examples of this higher usage:
1137          *      Level   Usage
1138          *      14      Profiling clock (and PROM uart polling clock)
1139          *      12      Serial ports
1140          *
1141          * The serial ports request lower level processing on level 6.
1142          *
1143          * Also, almost all splN routines (where N is a number or a
1144          * mnemonic) will do a RAISE(), on the assumption that they are
1145          * never used to lower our priority.
1146          * The exceptions are:
1147          *      spl8()          Because you can't be above 15 to begin with!
1148          *      splzs()         Because this is used at boot time to lower our
1149          *                      priority, to allow the PROM to poll the uart.
1150          *      spl0()          Used to lower priority to 0.
1151          */
1152 
1153 #if defined(__lint)
1154 
1155 int spl0(void)          { return (0); }
1156 int spl6(void)          { return (0); }
1157 int spl7(void)          { return (0); }
1158 int spl8(void)          { return (0); }
1159 int splhigh(void)       { return (0); }
1160 int splhi(void)         { return (0); }
1161 int splzs(void)         { return (0); }
1162 
1163 /* ARGSUSED */
1164 void
1165 splx(int level)
1166 {}
1167 
1168 #else   /* __lint */
1169 
1170 #if defined(__amd64)
1171 
1172 #define SETPRI(level) \
1173         movl    $/**/level, %edi;       /* new priority */              \
1174         jmp     do_splx                 /* redirect to do_splx */
1175 
1176 #define RAISE(level) \
1177         movl    $/**/level, %edi;       /* new priority */              \
1178         jmp     splr                    /* redirect to splr */
1179 
1180 #elif defined(__i386)
1181 
1182 #define SETPRI(level) \
1183         pushl   $/**/level;     /* new priority */                      \
1184         call    do_splx;        /* invoke common splx code */           \
1185         addl    $4, %esp;       /* unstack arg */                       \
1186         ret
1187 
1188 #define RAISE(level) \
1189         pushl   $/**/level;     /* new priority */                      \
1190         call    splr;           /* invoke common splr code */           \
1191         addl    $4, %esp;       /* unstack args */                      \
1192         ret
1193 
1194 #endif  /* __i386 */
1195 
1196         /* locks out all interrupts, including memory errors */
1197         ENTRY(spl8)
1198         SETPRI(15)
1199         SET_SIZE(spl8)
1200 
1201         /* just below the level that profiling runs */
1202         ENTRY(spl7)
1203         RAISE(13)
1204         SET_SIZE(spl7)
1205 
1206         /* sun specific - highest priority onboard serial i/o asy ports */
1207         ENTRY(splzs)
1208         SETPRI(12)      /* Can't be a RAISE, as it's used to lower us */
1209         SET_SIZE(splzs)
1210 
1211         ENTRY(splhi)
1212         ALTENTRY(splhigh)
1213         ALTENTRY(spl6)
1214         ALTENTRY(i_ddi_splhigh)
1215 
1216         RAISE(DISP_LEVEL)
1217 
1218         SET_SIZE(i_ddi_splhigh)
1219         SET_SIZE(spl6)
1220         SET_SIZE(splhigh)
1221         SET_SIZE(splhi)
1222 
1223         /* allow all interrupts */
1224         ENTRY(spl0)
1225         SETPRI(0)
1226         SET_SIZE(spl0)
1227 
1228 
1229         /* splx implementation */
1230         ENTRY(splx)
1231         jmp     do_splx         /* redirect to common splx code */
1232         SET_SIZE(splx)
1233 
1234 #endif  /* __lint */
1235 
1236 #if defined(__i386)
1237 
1238 /*
1239  * Read and write the %gs register
1240  */
1241 
1242 #if defined(__lint)
1243 
1244 /*ARGSUSED*/
1245 uint16_t
1246 getgs(void)
1247 { return (0); }
1248 
1249 /*ARGSUSED*/
1250 void
1251 setgs(uint16_t sel)
1252 {}
1253 
1254 #else   /* __lint */
1255 
1256         ENTRY(getgs)
1257         clr     %eax
1258         movw    %gs, %ax
1259         ret
1260         SET_SIZE(getgs)
1261 
1262         ENTRY(setgs)
1263         movw    4(%esp), %gs
1264         ret
1265         SET_SIZE(setgs)
1266 
1267 #endif  /* __lint */
1268 #endif  /* __i386 */
1269 
1270 #if defined(__lint)
1271 
1272 void
1273 pc_reset(void)
1274 {}
1275 
1276 void
1277 efi_reset(void)
1278 {}
1279 
1280 #else   /* __lint */
1281 
1282         ENTRY(wait_500ms)
1283 #if defined(__amd64)
1284         pushq   %rbx
1285 #elif defined(__i386)
1286         push    %ebx
1287 #endif
1288         movl    $50000, %ebx
1289 1:
1290         call    tenmicrosec
1291         decl    %ebx
1292         jnz     1b
1293 #if defined(__amd64)
1294         popq    %rbx
1295 #elif defined(__i386)
1296         pop     %ebx
1297 #endif
1298         ret     
1299         SET_SIZE(wait_500ms)
1300 
1301 #define RESET_METHOD_KBC        1
1302 #define RESET_METHOD_PORT92     2
1303 #define RESET_METHOD_PCI        4
1304 
1305         DGDEF3(pc_reset_methods, 4, 8)
1306         .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1307 
1308         ENTRY(pc_reset)
1309 
1310 #if defined(__i386)
1311         testl   $RESET_METHOD_KBC, pc_reset_methods
1312 #elif defined(__amd64)
1313         testl   $RESET_METHOD_KBC, pc_reset_methods(%rip)
1314 #endif
1315         jz      1f
1316 
1317         /
1318         / Try the classic keyboard controller-triggered reset.
1319         /
1320         movw    $0x64, %dx
1321         movb    $0xfe, %al
1322         outb    (%dx)
1323 
1324         / Wait up to 500 milliseconds here for the keyboard controller
1325         / to pull the reset line.  On some systems where the keyboard
1326         / controller is slow to pull the reset line, the next reset method
1327         / may be executed (which may be bad if those systems hang when the
1328         / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1329         / and Ferrari 4000 (doesn't like the cf9 reset method))
1330 
1331         call    wait_500ms
1332 
1333 1:
1334 #if defined(__i386)
1335         testl   $RESET_METHOD_PORT92, pc_reset_methods
1336 #elif defined(__amd64)
1337         testl   $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1338 #endif
1339         jz      3f
1340 
1341         /
1342         / Try port 0x92 fast reset
1343         /
1344         movw    $0x92, %dx
1345         inb     (%dx)
1346         cmpb    $0xff, %al      / If port's not there, we should get back 0xFF
1347         je      1f
1348         testb   $1, %al         / If bit 0
1349         jz      2f              / is clear, jump to perform the reset
1350         andb    $0xfe, %al      / otherwise,
1351         outb    (%dx)           / clear bit 0 first, then
1352 2:
1353         orb     $1, %al         / Set bit 0
1354         outb    (%dx)           / and reset the system
1355 1:
1356 
1357         call    wait_500ms
1358 
1359 3:
1360 #if defined(__i386)
1361         testl   $RESET_METHOD_PCI, pc_reset_methods
1362 #elif defined(__amd64)
1363         testl   $RESET_METHOD_PCI, pc_reset_methods(%rip)
1364 #endif
1365         jz      4f
1366 
1367         / Try the PCI (soft) reset vector (should work on all modern systems,
1368         / but has been shown to cause problems on 450NX systems, and some newer
1369         / systems (e.g. ATI IXP400-equipped systems))
1370         / When resetting via this method, 2 writes are required.  The first
1371         / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1372         / power cycle).
1373         / The reset occurs on the second write, during bit 2's transition from
1374         / 0->1.
1375         movw    $0xcf9, %dx
1376         movb    $0x2, %al       / Reset mode = hard, no power cycle
1377         outb    (%dx)
1378         movb    $0x6, %al
1379         outb    (%dx)
1380 
1381         call    wait_500ms
1382 
1383 4:
1384         /
1385         / port 0xcf9 failed also.  Last-ditch effort is to
1386         / triple-fault the CPU.
1387         / Also, use triple fault for EFI firmware
1388         /
1389         ENTRY(efi_reset)
1390 #if defined(__amd64)
1391         pushq   $0x0
1392         pushq   $0x0            / IDT base of 0, limit of 0 + 2 unused bytes
1393         lidt    (%rsp)
1394 #elif defined(__i386)
1395         pushl   $0x0
1396         pushl   $0x0            / IDT base of 0, limit of 0 + 2 unused bytes
1397         lidt    (%esp)
1398 #endif
1399         int     $0x0            / Trigger interrupt, generate triple-fault
1400 
1401         cli
1402         hlt                     / Wait forever
1403         /*NOTREACHED*/
1404         SET_SIZE(efi_reset)
1405         SET_SIZE(pc_reset)
1406 
1407 #endif  /* __lint */
1408 
1409 /*
1410  * C callable in and out routines
1411  */
1412 
1413 #if defined(__lint)
1414 
1415 /* ARGSUSED */
1416 void
1417 outl(int port_address, uint32_t val)
1418 {}
1419 
1420 #else   /* __lint */
1421 
1422 #if defined(__amd64)
1423 
1424         ENTRY(outl)
1425         movw    %di, %dx
1426         movl    %esi, %eax
1427         outl    (%dx)
1428         ret
1429         SET_SIZE(outl)
1430 
1431 #elif defined(__i386)
1432 
1433         .set    PORT, 4
1434         .set    VAL, 8
1435 
1436         ENTRY(outl)
1437         movw    PORT(%esp), %dx
1438         movl    VAL(%esp), %eax
1439         outl    (%dx)
1440         ret
1441         SET_SIZE(outl)
1442 
1443 #endif  /* __i386 */
1444 #endif  /* __lint */
1445 
1446 #if defined(__lint)
1447 
1448 /* ARGSUSED */
1449 void
1450 outw(int port_address, uint16_t val)
1451 {}
1452 
1453 #else   /* __lint */
1454 
1455 #if defined(__amd64)
1456 
1457         ENTRY(outw)
1458         movw    %di, %dx
1459         movw    %si, %ax
1460         D16 outl (%dx)          /* XX64 why not outw? */
1461         ret
1462         SET_SIZE(outw)
1463 
1464 #elif defined(__i386)
1465 
1466         ENTRY(outw)
1467         movw    PORT(%esp), %dx
1468         movw    VAL(%esp), %ax
1469         D16 outl (%dx)
1470         ret
1471         SET_SIZE(outw)
1472 
1473 #endif  /* __i386 */
1474 #endif  /* __lint */
1475 
1476 #if defined(__lint)
1477 
1478 /* ARGSUSED */
1479 void
1480 outb(int port_address, uint8_t val)
1481 {}
1482 
1483 #else   /* __lint */
1484 
1485 #if defined(__amd64)
1486 
1487         ENTRY(outb)
1488         movw    %di, %dx
1489         movb    %sil, %al
1490         outb    (%dx)
1491         ret
1492         SET_SIZE(outb)
1493 
1494 #elif defined(__i386)
1495 
1496         ENTRY(outb)
1497         movw    PORT(%esp), %dx
1498         movb    VAL(%esp), %al
1499         outb    (%dx)
1500         ret
1501         SET_SIZE(outb)
1502 
1503 #endif  /* __i386 */
1504 #endif  /* __lint */
1505 
1506 #if defined(__lint)
1507 
1508 /* ARGSUSED */
1509 uint32_t
1510 inl(int port_address)
1511 { return (0); }
1512 
1513 #else   /* __lint */
1514 
1515 #if defined(__amd64)
1516 
1517         ENTRY(inl)
1518         xorl    %eax, %eax
1519         movw    %di, %dx
1520         inl     (%dx)
1521         ret
1522         SET_SIZE(inl)
1523 
1524 #elif defined(__i386)
1525 
1526         ENTRY(inl)
1527         movw    PORT(%esp), %dx
1528         inl     (%dx)
1529         ret
1530         SET_SIZE(inl)
1531 
1532 #endif  /* __i386 */
1533 #endif  /* __lint */
1534 
1535 #if defined(__lint)
1536 
1537 /* ARGSUSED */
1538 uint16_t
1539 inw(int port_address)
1540 { return (0); }
1541 
1542 #else   /* __lint */
1543 
1544 #if defined(__amd64)
1545 
1546         ENTRY(inw)
1547         xorl    %eax, %eax
1548         movw    %di, %dx
1549         D16 inl (%dx)
1550         ret
1551         SET_SIZE(inw)
1552 
1553 #elif defined(__i386)
1554 
1555         ENTRY(inw)
1556         subl    %eax, %eax
1557         movw    PORT(%esp), %dx
1558         D16 inl (%dx)
1559         ret
1560         SET_SIZE(inw)
1561 
1562 #endif  /* __i386 */
1563 #endif  /* __lint */
1564 
1565 
1566 #if defined(__lint)
1567 
1568 /* ARGSUSED */
1569 uint8_t
1570 inb(int port_address)
1571 { return (0); }
1572 
1573 #else   /* __lint */
1574 
1575 #if defined(__amd64)
1576 
1577         ENTRY(inb)
1578         xorl    %eax, %eax
1579         movw    %di, %dx
1580         inb     (%dx)
1581         ret
1582         SET_SIZE(inb)
1583 
1584 #elif defined(__i386)
1585 
1586         ENTRY(inb)
1587         subl    %eax, %eax
1588         movw    PORT(%esp), %dx
1589         inb     (%dx)
1590         ret
1591         SET_SIZE(inb)
1592 
1593 #endif  /* __i386 */
1594 #endif  /* __lint */
1595 
1596 
1597 #if defined(__lint)
1598 
1599 /* ARGSUSED */
1600 void
1601 repoutsw(int port, uint16_t *addr, int cnt)
1602 {}
1603 
1604 #else   /* __lint */
1605 
1606 #if defined(__amd64)
1607 
1608         ENTRY(repoutsw)
1609         movl    %edx, %ecx
1610         movw    %di, %dx
1611         rep
1612           D16 outsl
1613         ret
1614         SET_SIZE(repoutsw)
1615 
1616 #elif defined(__i386)
1617 
1618         /*
1619          * The arguments and saved registers are on the stack in the
1620          *  following order:
1621          *      |  cnt  |  +16
1622          *      | *addr |  +12
1623          *      | port  |  +8
1624          *      |  eip  |  +4
1625          *      |  esi  |  <-- %esp
1626          * If additional values are pushed onto the stack, make sure
1627          * to adjust the following constants accordingly.
1628          */
1629         .set    PORT, 8
1630         .set    ADDR, 12
1631         .set    COUNT, 16
1632 
1633         ENTRY(repoutsw)
1634         pushl   %esi
1635         movl    PORT(%esp), %edx
1636         movl    ADDR(%esp), %esi
1637         movl    COUNT(%esp), %ecx
1638         rep
1639           D16 outsl
1640         popl    %esi
1641         ret
1642         SET_SIZE(repoutsw)
1643 
1644 #endif  /* __i386 */
1645 #endif  /* __lint */
1646 
1647 
1648 #if defined(__lint)
1649 
1650 /* ARGSUSED */
1651 void
1652 repinsw(int port_addr, uint16_t *addr, int cnt)
1653 {}
1654 
1655 #else   /* __lint */
1656 
1657 #if defined(__amd64)
1658 
1659         ENTRY(repinsw)
1660         movl    %edx, %ecx
1661         movw    %di, %dx
1662         rep
1663           D16 insl
1664         ret
1665         SET_SIZE(repinsw)
1666 
1667 #elif defined(__i386)
1668 
1669         ENTRY(repinsw)
1670         pushl   %edi
1671         movl    PORT(%esp), %edx
1672         movl    ADDR(%esp), %edi
1673         movl    COUNT(%esp), %ecx
1674         rep
1675           D16 insl
1676         popl    %edi
1677         ret
1678         SET_SIZE(repinsw)
1679 
1680 #endif  /* __i386 */
1681 #endif  /* __lint */
1682 
1683 
1684 #if defined(__lint)
1685 
1686 /* ARGSUSED */
1687 void
1688 repinsb(int port, uint8_t *addr, int count)
1689 {}
1690 
1691 #else   /* __lint */
1692 
1693 #if defined(__amd64)
1694 
1695         ENTRY(repinsb)
1696         movl    %edx, %ecx      
1697         movw    %di, %dx
1698         movq    %rsi, %rdi
1699         rep
1700           insb
1701         ret             
1702         SET_SIZE(repinsb)
1703 
1704 #elif defined(__i386)
1705         
1706         /*
1707          * The arguments and saved registers are on the stack in the
1708          *  following order:
1709          *      |  cnt  |  +16
1710          *      | *addr |  +12
1711          *      | port  |  +8
1712          *      |  eip  |  +4
1713          *      |  esi  |  <-- %esp
1714          * If additional values are pushed onto the stack, make sure
1715          * to adjust the following constants accordingly.
1716          */
1717         .set    IO_PORT, 8
1718         .set    IO_ADDR, 12
1719         .set    IO_COUNT, 16
1720 
1721         ENTRY(repinsb)
1722         pushl   %edi
1723         movl    IO_ADDR(%esp), %edi
1724         movl    IO_COUNT(%esp), %ecx
1725         movl    IO_PORT(%esp), %edx
1726         rep
1727           insb
1728         popl    %edi
1729         ret
1730         SET_SIZE(repinsb)
1731 
1732 #endif  /* __i386 */
1733 #endif  /* __lint */
1734 
1735 
1736 /*
1737  * Input a stream of 32-bit words.
1738  * NOTE: count is a DWORD count.
1739  */
1740 #if defined(__lint)
1741 
1742 /* ARGSUSED */
1743 void
1744 repinsd(int port, uint32_t *addr, int count)
1745 {}
1746 
1747 #else   /* __lint */
1748 
1749 #if defined(__amd64)
1750         
1751         ENTRY(repinsd)
1752         movl    %edx, %ecx
1753         movw    %di, %dx
1754         movq    %rsi, %rdi
1755         rep
1756           insl
1757         ret
1758         SET_SIZE(repinsd)
1759 
1760 #elif defined(__i386)
1761 
1762         ENTRY(repinsd)
1763         pushl   %edi
1764         movl    IO_ADDR(%esp), %edi
1765         movl    IO_COUNT(%esp), %ecx
1766         movl    IO_PORT(%esp), %edx
1767         rep
1768           insl
1769         popl    %edi
1770         ret
1771         SET_SIZE(repinsd)
1772 
1773 #endif  /* __i386 */
1774 #endif  /* __lint */
1775 
1776 /*
1777  * Output a stream of bytes
1778  * NOTE: count is a byte count
1779  */
1780 #if defined(__lint)
1781 
1782 /* ARGSUSED */
1783 void
1784 repoutsb(int port, uint8_t *addr, int count)
1785 {}
1786 
1787 #else   /* __lint */
1788 
1789 #if defined(__amd64)
1790 
1791         ENTRY(repoutsb)
1792         movl    %edx, %ecx
1793         movw    %di, %dx
1794         rep
1795           outsb
1796         ret     
1797         SET_SIZE(repoutsb)
1798 
1799 #elif defined(__i386)
1800 
1801         ENTRY(repoutsb)
1802         pushl   %esi
1803         movl    IO_ADDR(%esp), %esi
1804         movl    IO_COUNT(%esp), %ecx
1805         movl    IO_PORT(%esp), %edx
1806         rep
1807           outsb
1808         popl    %esi
1809         ret
1810         SET_SIZE(repoutsb)
1811 
1812 #endif  /* __i386 */    
1813 #endif  /* __lint */
1814 
1815 /*
1816  * Output a stream of 32-bit words
1817  * NOTE: count is a DWORD count
1818  */
1819 #if defined(__lint)
1820 
1821 /* ARGSUSED */
1822 void
1823 repoutsd(int port, uint32_t *addr, int count)
1824 {}
1825 
1826 #else   /* __lint */
1827 
1828 #if defined(__amd64)
1829 
1830         ENTRY(repoutsd)
1831         movl    %edx, %ecx
1832         movw    %di, %dx
1833         rep
1834           outsl
1835         ret     
1836         SET_SIZE(repoutsd)
1837 
1838 #elif defined(__i386)
1839 
1840         ENTRY(repoutsd)
1841         pushl   %esi
1842         movl    IO_ADDR(%esp), %esi
1843         movl    IO_COUNT(%esp), %ecx
1844         movl    IO_PORT(%esp), %edx
1845         rep
1846           outsl
1847         popl    %esi
1848         ret
1849         SET_SIZE(repoutsd)
1850 
1851 #endif  /* __i386 */
1852 #endif  /* __lint */
1853 
1854 /*
1855  * void int3(void)
1856  * void int18(void)
1857  * void int20(void)
1858  * void int_cmci(void)
1859  */
1860 
1861 #if defined(__lint)
1862 
1863 void
1864 int3(void)
1865 {}
1866 
1867 void
1868 int18(void)
1869 {}
1870 
1871 void
1872 int20(void)
1873 {}
1874 
1875 void
1876 int_cmci(void)
1877 {}
1878 
1879 #else   /* __lint */
1880 
1881         ENTRY(int3)
1882         int     $T_BPTFLT
1883         ret
1884         SET_SIZE(int3)
1885 
1886         ENTRY(int18)
1887         int     $T_MCE
1888         ret
1889         SET_SIZE(int18)
1890 
1891         ENTRY(int20)
1892         movl    boothowto, %eax
1893         andl    $RB_DEBUG, %eax
1894         jz      1f
1895 
1896         int     $T_DBGENTR
1897 1:
1898         rep;    ret     /* use 2 byte return instruction when branch target */
1899                         /* AMD Software Optimization Guide - Section 6.2 */
1900         SET_SIZE(int20)
1901 
1902         ENTRY(int_cmci)
1903         int     $T_ENOEXTFLT
1904         ret
1905         SET_SIZE(int_cmci)
1906 
1907 #endif  /* __lint */
1908 
1909 #if defined(__lint)
1910 
1911 /* ARGSUSED */
1912 int
1913 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1914 { return (0); }
1915 
1916 #else   /* __lint */
1917 
1918 #if defined(__amd64)
1919 
1920         ENTRY(scanc)
1921                                         /* rdi == size */
1922                                         /* rsi == cp */
1923                                         /* rdx == table */
1924                                         /* rcx == mask */
1925         addq    %rsi, %rdi              /* end = &cp[size] */
1926 .scanloop:
1927         cmpq    %rdi, %rsi              /* while (cp < end */
1928         jnb     .scandone
1929         movzbq  (%rsi), %r8             /* %r8 = *cp */
1930         incq    %rsi                    /* cp++ */
1931         testb   %cl, (%r8, %rdx)
1932         jz      .scanloop               /*  && (table[*cp] & mask) == 0) */
1933         decq    %rsi                    /* (fix post-increment) */
1934 .scandone:
1935         movl    %edi, %eax
1936         subl    %esi, %eax              /* return (end - cp) */
1937         ret
1938         SET_SIZE(scanc)
1939 
1940 #elif defined(__i386)
1941 
1942         ENTRY(scanc)
1943         pushl   %edi
1944         pushl   %esi
1945         movb    24(%esp), %cl           /* mask = %cl */
1946         movl    16(%esp), %esi          /* cp = %esi */
1947         movl    20(%esp), %edx          /* table = %edx */
1948         movl    %esi, %edi
1949         addl    12(%esp), %edi          /* end = &cp[size]; */
1950 .scanloop:
1951         cmpl    %edi, %esi              /* while (cp < end */
1952         jnb     .scandone
1953         movzbl  (%esi),  %eax           /* %al = *cp */
1954         incl    %esi                    /* cp++ */
1955         movb    (%edx,  %eax), %al      /* %al = table[*cp] */
1956         testb   %al, %cl
1957         jz      .scanloop               /*   && (table[*cp] & mask) == 0) */
1958         dec     %esi                    /* post-incremented */
1959 .scandone:
1960         movl    %edi, %eax
1961         subl    %esi, %eax              /* return (end - cp) */
1962         popl    %esi
1963         popl    %edi
1964         ret
1965         SET_SIZE(scanc)
1966 
1967 #endif  /* __i386 */
1968 #endif  /* __lint */
1969 
1970 /*
1971  * Replacement functions for ones that are normally inlined.
1972  * In addition to the copy in i86.il, they are defined here just in case.
1973  */
1974 
1975 #if defined(__lint)
1976 
1977 ulong_t
1978 intr_clear(void)
1979 { return (0); }
1980 
1981 ulong_t
1982 clear_int_flag(void)
1983 { return (0); }
1984 
1985 #else   /* __lint */
1986 
1987 #if defined(__amd64)
1988 
1989         ENTRY(intr_clear)
1990         ENTRY(clear_int_flag)
1991         pushfq
1992         popq    %rax
1993 #if defined(__xpv)
1994         leaq    xpv_panicking, %rdi
1995         movl    (%rdi), %edi
1996         cmpl    $0, %edi
1997         jne     2f
1998         CLIRET(%rdi, %dl)       /* returns event mask in %dl */
1999         /*
2000          * Synthesize the PS_IE bit from the event mask bit
2001          */
2002         andq    $_BITNOT(PS_IE), %rax
2003         testb   $1, %dl
2004         jnz     1f
2005         orq     $PS_IE, %rax
2006 1:
2007         ret
2008 2:
2009 #endif
2010         CLI(%rdi)
2011         ret
2012         SET_SIZE(clear_int_flag)
2013         SET_SIZE(intr_clear)
2014 
2015 #elif defined(__i386)
2016 
2017         ENTRY(intr_clear)
2018         ENTRY(clear_int_flag)
2019         pushfl
2020         popl    %eax
2021 #if defined(__xpv)
2022         leal    xpv_panicking, %edx
2023         movl    (%edx), %edx
2024         cmpl    $0, %edx
2025         jne     2f
2026         CLIRET(%edx, %cl)       /* returns event mask in %cl */
2027         /*
2028          * Synthesize the PS_IE bit from the event mask bit
2029          */
2030         andl    $_BITNOT(PS_IE), %eax
2031         testb   $1, %cl
2032         jnz     1f
2033         orl     $PS_IE, %eax
2034 1:
2035         ret
2036 2:
2037 #endif
2038         CLI(%edx)
2039         ret
2040         SET_SIZE(clear_int_flag)
2041         SET_SIZE(intr_clear)
2042 
2043 #endif  /* __i386 */
2044 #endif  /* __lint */
2045 
2046 #if defined(__lint)
2047 
2048 struct cpu *
2049 curcpup(void)
2050 { return 0; }
2051 
2052 #else   /* __lint */
2053 
2054 #if defined(__amd64)
2055 
2056         ENTRY(curcpup)
2057         movq    %gs:CPU_SELF, %rax
2058         ret
2059         SET_SIZE(curcpup)
2060 
2061 #elif defined(__i386)
2062 
2063         ENTRY(curcpup)
2064         movl    %gs:CPU_SELF, %eax
2065         ret
2066         SET_SIZE(curcpup)
2067 
2068 #endif  /* __i386 */
2069 #endif  /* __lint */
2070 
2071 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2072  * These functions reverse the byte order of the input parameter and returns
2073  * the result.  This is to convert the byte order from host byte order
2074  * (little endian) to network byte order (big endian), or vice versa.
2075  */
2076 
2077 #if defined(__lint)
2078 
2079 uint64_t
2080 htonll(uint64_t i)
2081 { return (i); }
2082 
2083 uint64_t
2084 ntohll(uint64_t i)
2085 { return (i); }
2086 
2087 uint32_t
2088 htonl(uint32_t i)
2089 { return (i); }
2090 
2091 uint32_t
2092 ntohl(uint32_t i)
2093 { return (i); }
2094 
2095 uint16_t
2096 htons(uint16_t i)
2097 { return (i); }
2098 
2099 uint16_t
2100 ntohs(uint16_t i)
2101 { return (i); }
2102 
2103 #else   /* __lint */
2104 
2105 #if defined(__amd64)
2106 
2107         ENTRY(htonll)
2108         ALTENTRY(ntohll)
2109         movq    %rdi, %rax
2110         bswapq  %rax
2111         ret
2112         SET_SIZE(ntohll)
2113         SET_SIZE(htonll)
2114 
2115         /* XX64 there must be shorter sequences for this */
2116         ENTRY(htonl)
2117         ALTENTRY(ntohl)
2118         movl    %edi, %eax
2119         bswap   %eax
2120         ret
2121         SET_SIZE(ntohl)
2122         SET_SIZE(htonl)
2123 
2124         /* XX64 there must be better sequences for this */
2125         ENTRY(htons)
2126         ALTENTRY(ntohs)
2127         movl    %edi, %eax
2128         bswap   %eax
2129         shrl    $16, %eax
2130         ret
2131         SET_SIZE(ntohs)
2132         SET_SIZE(htons)
2133 
2134 #elif defined(__i386)
2135 
2136         ENTRY(htonll)
2137         ALTENTRY(ntohll)
2138         movl    4(%esp), %edx
2139         movl    8(%esp), %eax
2140         bswap   %edx
2141         bswap   %eax
2142         ret
2143         SET_SIZE(ntohll)
2144         SET_SIZE(htonll)
2145 
2146         ENTRY(htonl)
2147         ALTENTRY(ntohl)
2148         movl    4(%esp), %eax
2149         bswap   %eax
2150         ret
2151         SET_SIZE(ntohl)
2152         SET_SIZE(htonl)
2153 
2154         ENTRY(htons)
2155         ALTENTRY(ntohs)
2156         movl    4(%esp), %eax
2157         bswap   %eax
2158         shrl    $16, %eax
2159         ret
2160         SET_SIZE(ntohs)
2161         SET_SIZE(htons)
2162 
2163 #endif  /* __i386 */
2164 #endif  /* __lint */
2165 
2166 
2167 #if defined(__lint)
2168 
2169 /* ARGSUSED */
2170 void
2171 intr_restore(ulong_t i)
2172 { return; }
2173 
2174 /* ARGSUSED */
2175 void
2176 restore_int_flag(ulong_t i)
2177 { return; }
2178 
2179 #else   /* __lint */
2180 
2181 #if defined(__amd64)
2182 
2183         ENTRY(intr_restore)
2184         ENTRY(restore_int_flag)
2185         testq   $PS_IE, %rdi
2186         jz      1f
2187 #if defined(__xpv)
2188         leaq    xpv_panicking, %rsi
2189         movl    (%rsi), %esi
2190         cmpl    $0, %esi
2191         jne     1f
2192         /*
2193          * Since we're -really- running unprivileged, our attempt
2194          * to change the state of the IF bit will be ignored.
2195          * The virtual IF bit is tweaked by CLI and STI.
2196          */
2197         IE_TO_EVENT_MASK(%rsi, %rdi)
2198 #else
2199         sti
2200 #endif
2201 1:
2202         ret
2203         SET_SIZE(restore_int_flag)
2204         SET_SIZE(intr_restore)
2205 
2206 #elif defined(__i386)
2207 
2208         ENTRY(intr_restore)
2209         ENTRY(restore_int_flag)
2210         testl   $PS_IE, 4(%esp)
2211         jz      1f
2212 #if defined(__xpv)
2213         leal    xpv_panicking, %edx
2214         movl    (%edx), %edx
2215         cmpl    $0, %edx
2216         jne     1f
2217         /*
2218          * Since we're -really- running unprivileged, our attempt
2219          * to change the state of the IF bit will be ignored.
2220          * The virtual IF bit is tweaked by CLI and STI.
2221          */
2222         IE_TO_EVENT_MASK(%edx, 4(%esp))
2223 #else
2224         sti
2225 #endif
2226 1:
2227         ret
2228         SET_SIZE(restore_int_flag)
2229         SET_SIZE(intr_restore)
2230 
2231 #endif  /* __i386 */
2232 #endif  /* __lint */
2233 
2234 #if defined(__lint)
2235 
2236 void
2237 sti(void)
2238 {}
2239 
2240 void
2241 cli(void)
2242 {}
2243 
2244 #else   /* __lint */
2245 
2246         ENTRY(sti)
2247         STI
2248         ret
2249         SET_SIZE(sti)
2250 
2251         ENTRY(cli)
2252 #if defined(__amd64)
2253         CLI(%rax)
2254 #elif defined(__i386)
2255         CLI(%eax)
2256 #endif  /* __i386 */
2257         ret
2258         SET_SIZE(cli)
2259 
2260 #endif  /* __lint */
2261 
2262 #if defined(__lint)
2263 
2264 dtrace_icookie_t
2265 dtrace_interrupt_disable(void)
2266 { return (0); }
2267 
2268 #else   /* __lint */
2269 
2270 #if defined(__amd64)
2271 
2272         ENTRY(dtrace_interrupt_disable)
2273         pushfq
2274         popq    %rax
2275 #if defined(__xpv)
2276         leaq    xpv_panicking, %rdi
2277         movl    (%rdi), %edi
2278         cmpl    $0, %edi
2279         jne     .dtrace_interrupt_disable_done
2280         CLIRET(%rdi, %dl)       /* returns event mask in %dl */
2281         /*
2282          * Synthesize the PS_IE bit from the event mask bit
2283          */
2284         andq    $_BITNOT(PS_IE), %rax
2285         testb   $1, %dl
2286         jnz     .dtrace_interrupt_disable_done
2287         orq     $PS_IE, %rax
2288 #else
2289         CLI(%rdx)
2290 #endif
2291 .dtrace_interrupt_disable_done:
2292         ret
2293         SET_SIZE(dtrace_interrupt_disable)
2294 
2295 #elif defined(__i386)
2296                 
2297         ENTRY(dtrace_interrupt_disable)
2298         pushfl
2299         popl    %eax
2300 #if defined(__xpv)
2301         leal    xpv_panicking, %edx
2302         movl    (%edx), %edx
2303         cmpl    $0, %edx
2304         jne     .dtrace_interrupt_disable_done
2305         CLIRET(%edx, %cl)       /* returns event mask in %cl */
2306         /*
2307          * Synthesize the PS_IE bit from the event mask bit
2308          */
2309         andl    $_BITNOT(PS_IE), %eax
2310         testb   $1, %cl
2311         jnz     .dtrace_interrupt_disable_done
2312         orl     $PS_IE, %eax
2313 #else
2314         CLI(%edx)
2315 #endif
2316 .dtrace_interrupt_disable_done:
2317         ret
2318         SET_SIZE(dtrace_interrupt_disable)
2319 
2320 #endif  /* __i386 */    
2321 #endif  /* __lint */
2322 
2323 #if defined(__lint)
2324 
2325 /*ARGSUSED*/
2326 void
2327 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2328 {}
2329 
2330 #else   /* __lint */
2331 
2332 #if defined(__amd64)
2333 
2334         ENTRY(dtrace_interrupt_enable)
2335         pushq   %rdi
2336         popfq
2337 #if defined(__xpv)
2338         leaq    xpv_panicking, %rdx
2339         movl    (%rdx), %edx
2340         cmpl    $0, %edx
2341         jne     .dtrace_interrupt_enable_done
2342         /*
2343          * Since we're -really- running unprivileged, our attempt
2344          * to change the state of the IF bit will be ignored. The
2345          * virtual IF bit is tweaked by CLI and STI.
2346          */
2347         IE_TO_EVENT_MASK(%rdx, %rdi)
2348 #endif
2349 .dtrace_interrupt_enable_done:
2350         ret
2351         SET_SIZE(dtrace_interrupt_enable)
2352 
2353 #elif defined(__i386)
2354                 
2355         ENTRY(dtrace_interrupt_enable)
2356         movl    4(%esp), %eax
2357         pushl   %eax
2358         popfl
2359 #if defined(__xpv)
2360         leal    xpv_panicking, %edx
2361         movl    (%edx), %edx
2362         cmpl    $0, %edx
2363         jne     .dtrace_interrupt_enable_done
2364         /*
2365          * Since we're -really- running unprivileged, our attempt
2366          * to change the state of the IF bit will be ignored. The
2367          * virtual IF bit is tweaked by CLI and STI.
2368          */
2369         IE_TO_EVENT_MASK(%edx, %eax)
2370 #endif
2371 .dtrace_interrupt_enable_done:
2372         ret
2373         SET_SIZE(dtrace_interrupt_enable)
2374 
2375 #endif  /* __i386 */    
2376 #endif  /* __lint */
2377 
2378 
2379 #if defined(lint)
2380 
2381 void
2382 dtrace_membar_producer(void)
2383 {}
2384 
2385 void
2386 dtrace_membar_consumer(void)
2387 {}
2388 
2389 #else   /* __lint */
2390 
2391         ENTRY(dtrace_membar_producer)
2392         rep;    ret     /* use 2 byte return instruction when branch target */
2393                         /* AMD Software Optimization Guide - Section 6.2 */
2394         SET_SIZE(dtrace_membar_producer)
2395 
2396         ENTRY(dtrace_membar_consumer)
2397         rep;    ret     /* use 2 byte return instruction when branch target */
2398                         /* AMD Software Optimization Guide - Section 6.2 */
2399         SET_SIZE(dtrace_membar_consumer)
2400 
2401 #endif  /* __lint */
2402 
2403 #if defined(__lint)
2404 
2405 kthread_id_t
2406 threadp(void)
2407 { return ((kthread_id_t)0); }
2408 
2409 #else   /* __lint */
2410 
2411 #if defined(__amd64)
2412         
2413         ENTRY(threadp)
2414         movq    %gs:CPU_THREAD, %rax
2415         ret
2416         SET_SIZE(threadp)
2417 
2418 #elif defined(__i386)
2419 
2420         ENTRY(threadp)
2421         movl    %gs:CPU_THREAD, %eax
2422         ret
2423         SET_SIZE(threadp)
2424 
2425 #endif  /* __i386 */
2426 #endif  /* __lint */
2427 
2428 /*
2429  *   Checksum routine for Internet Protocol Headers
2430  */
2431 
2432 #if defined(__lint)
2433 
2434 /* ARGSUSED */
2435 unsigned int
2436 ip_ocsum(
2437         ushort_t *address,      /* ptr to 1st message buffer */
2438         int halfword_count,     /* length of data */
2439         unsigned int sum)       /* partial checksum */
2440 { 
2441         int             i;
2442         unsigned int    psum = 0;       /* partial sum */
2443 
2444         for (i = 0; i < halfword_count; i++, address++) {
2445                 psum += *address;
2446         }
2447 
2448         while ((psum >> 16) != 0) {
2449                 psum = (psum & 0xffff) + (psum >> 16);
2450         }
2451 
2452         psum += sum;
2453 
2454         while ((psum >> 16) != 0) {
2455                 psum = (psum & 0xffff) + (psum >> 16);
2456         }
2457 
2458         return (psum);
2459 }
2460 
2461 #else   /* __lint */
2462 
2463 #if defined(__amd64)
2464 
2465         ENTRY(ip_ocsum)
2466         pushq   %rbp
2467         movq    %rsp, %rbp
2468 #ifdef DEBUG
2469         movq    postbootkernelbase(%rip), %rax
2470         cmpq    %rax, %rdi
2471         jnb     1f
2472         xorl    %eax, %eax
2473         movq    %rdi, %rsi
2474         leaq    .ip_ocsum_panic_msg(%rip), %rdi
2475         call    panic
2476         /*NOTREACHED*/
2477 .ip_ocsum_panic_msg:
2478         .string "ip_ocsum: address 0x%p below kernelbase\n"
2479 1:
2480 #endif
2481         movl    %esi, %ecx      /* halfword_count */
2482         movq    %rdi, %rsi      /* address */
2483                                 /* partial sum in %edx */
2484         xorl    %eax, %eax
2485         testl   %ecx, %ecx
2486         jz      .ip_ocsum_done
2487         testq   $3, %rsi
2488         jnz     .ip_csum_notaligned
2489 .ip_csum_aligned:       /* XX64 opportunities for 8-byte operations? */
2490 .next_iter:
2491         /* XX64 opportunities for prefetch? */
2492         /* XX64 compute csum with 64 bit quantities? */
2493         subl    $32, %ecx
2494         jl      .less_than_32
2495 
2496         addl    0(%rsi), %edx
2497 .only60:
2498         adcl    4(%rsi), %eax
2499 .only56:
2500         adcl    8(%rsi), %edx
2501 .only52:
2502         adcl    12(%rsi), %eax
2503 .only48:
2504         adcl    16(%rsi), %edx
2505 .only44:
2506         adcl    20(%rsi), %eax
2507 .only40:
2508         adcl    24(%rsi), %edx
2509 .only36:
2510         adcl    28(%rsi), %eax
2511 .only32:
2512         adcl    32(%rsi), %edx
2513 .only28:
2514         adcl    36(%rsi), %eax
2515 .only24:
2516         adcl    40(%rsi), %edx
2517 .only20:
2518         adcl    44(%rsi), %eax
2519 .only16:
2520         adcl    48(%rsi), %edx
2521 .only12:
2522         adcl    52(%rsi), %eax
2523 .only8:
2524         adcl    56(%rsi), %edx
2525 .only4:
2526         adcl    60(%rsi), %eax  /* could be adding -1 and -1 with a carry */
2527 .only0:
2528         adcl    $0, %eax        /* could be adding -1 in eax with a carry */
2529         adcl    $0, %eax
2530 
2531         addq    $64, %rsi
2532         testl   %ecx, %ecx
2533         jnz     .next_iter
2534 
2535 .ip_ocsum_done:
2536         addl    %eax, %edx
2537         adcl    $0, %edx
2538         movl    %edx, %eax      /* form a 16 bit checksum by */
2539         shrl    $16, %eax       /* adding two halves of 32 bit checksum */
2540         addw    %dx, %ax
2541         adcw    $0, %ax
2542         andl    $0xffff, %eax
2543         leave
2544         ret
2545 
2546 .ip_csum_notaligned:
2547         xorl    %edi, %edi
2548         movw    (%rsi), %di
2549         addl    %edi, %edx
2550         adcl    $0, %edx
2551         addq    $2, %rsi
2552         decl    %ecx
2553         jmp     .ip_csum_aligned
2554 
2555 .less_than_32:
2556         addl    $32, %ecx
2557         testl   $1, %ecx
2558         jz      .size_aligned
2559         andl    $0xfe, %ecx
2560         movzwl  (%rsi, %rcx, 2), %edi
2561         addl    %edi, %edx
2562         adcl    $0, %edx
2563 .size_aligned:
2564         movl    %ecx, %edi
2565         shrl    $1, %ecx
2566         shl     $1, %edi
2567         subq    $64, %rdi
2568         addq    %rdi, %rsi
2569         leaq    .ip_ocsum_jmptbl(%rip), %rdi
2570         leaq    (%rdi, %rcx, 8), %rdi
2571         xorl    %ecx, %ecx
2572         clc
2573         jmp     *(%rdi)
2574 
2575         .align  8
2576 .ip_ocsum_jmptbl:
2577         .quad   .only0, .only4, .only8, .only12, .only16, .only20
2578         .quad   .only24, .only28, .only32, .only36, .only40, .only44
2579         .quad   .only48, .only52, .only56, .only60
2580         SET_SIZE(ip_ocsum)
2581 
2582 #elif defined(__i386)
2583 
2584         ENTRY(ip_ocsum)
2585         pushl   %ebp
2586         movl    %esp, %ebp
2587         pushl   %ebx
2588         pushl   %esi
2589         pushl   %edi
2590         movl    12(%ebp), %ecx  /* count of half words */
2591         movl    16(%ebp), %edx  /* partial checksum */
2592         movl    8(%ebp), %esi
2593         xorl    %eax, %eax
2594         testl   %ecx, %ecx
2595         jz      .ip_ocsum_done
2596 
2597         testl   $3, %esi
2598         jnz     .ip_csum_notaligned
2599 .ip_csum_aligned:
2600 .next_iter:
2601         subl    $32, %ecx
2602         jl      .less_than_32
2603 
2604         addl    0(%esi), %edx
2605 .only60:
2606         adcl    4(%esi), %eax
2607 .only56:
2608         adcl    8(%esi), %edx
2609 .only52:
2610         adcl    12(%esi), %eax
2611 .only48:
2612         adcl    16(%esi), %edx
2613 .only44:
2614         adcl    20(%esi), %eax
2615 .only40:
2616         adcl    24(%esi), %edx
2617 .only36:
2618         adcl    28(%esi), %eax
2619 .only32:
2620         adcl    32(%esi), %edx
2621 .only28:
2622         adcl    36(%esi), %eax
2623 .only24:
2624         adcl    40(%esi), %edx
2625 .only20:
2626         adcl    44(%esi), %eax
2627 .only16:
2628         adcl    48(%esi), %edx
2629 .only12:
2630         adcl    52(%esi), %eax
2631 .only8:
2632         adcl    56(%esi), %edx
2633 .only4:
2634         adcl    60(%esi), %eax  /* We could be adding -1 and -1 with a carry */
2635 .only0:
2636         adcl    $0, %eax        /* we could be adding -1 in eax with a carry */
2637         adcl    $0, %eax
2638 
2639         addl    $64, %esi
2640         andl    %ecx, %ecx
2641         jnz     .next_iter
2642 
2643 .ip_ocsum_done:
2644         addl    %eax, %edx
2645         adcl    $0, %edx
2646         movl    %edx, %eax      /* form a 16 bit checksum by */
2647         shrl    $16, %eax       /* adding two halves of 32 bit checksum */
2648         addw    %dx, %ax
2649         adcw    $0, %ax
2650         andl    $0xffff, %eax
2651         popl    %edi            /* restore registers */
2652         popl    %esi
2653         popl    %ebx
2654         leave
2655         ret
2656 
2657 .ip_csum_notaligned:
2658         xorl    %edi, %edi
2659         movw    (%esi), %di
2660         addl    %edi, %edx
2661         adcl    $0, %edx
2662         addl    $2, %esi
2663         decl    %ecx
2664         jmp     .ip_csum_aligned
2665 
2666 .less_than_32:
2667         addl    $32, %ecx
2668         testl   $1, %ecx
2669         jz      .size_aligned
2670         andl    $0xfe, %ecx
2671         movzwl  (%esi, %ecx, 2), %edi
2672         addl    %edi, %edx
2673         adcl    $0, %edx
2674 .size_aligned:
2675         movl    %ecx, %edi
2676         shrl    $1, %ecx
2677         shl     $1, %edi
2678         subl    $64, %edi
2679         addl    %edi, %esi
2680         movl    $.ip_ocsum_jmptbl, %edi
2681         lea     (%edi, %ecx, 4), %edi
2682         xorl    %ecx, %ecx
2683         clc
2684         jmp     *(%edi)
2685         SET_SIZE(ip_ocsum)
2686 
2687         .data
2688         .align  4
2689 
2690 .ip_ocsum_jmptbl:
2691         .long   .only0, .only4, .only8, .only12, .only16, .only20
2692         .long   .only24, .only28, .only32, .only36, .only40, .only44
2693         .long   .only48, .only52, .only56, .only60
2694 
2695         
2696 #endif  /* __i386 */            
2697 #endif  /* __lint */
2698 
2699 /*
2700  * multiply two long numbers and yield a u_longlong_t result, callable from C.
2701  * Provided to manipulate hrtime_t values.
2702  */
2703 #if defined(__lint)
2704 
2705 /* result = a * b; */
2706 
2707 /* ARGSUSED */
2708 unsigned long long
2709 mul32(uint_t a, uint_t b)
2710 { return (0); }
2711 
2712 #else   /* __lint */
2713 
2714 #if defined(__amd64)
2715 
2716         ENTRY(mul32)
2717         xorl    %edx, %edx      /* XX64 joe, paranoia? */
2718         movl    %edi, %eax
2719         mull    %esi
2720         shlq    $32, %rdx       
2721         orq     %rdx, %rax
2722         ret
2723         SET_SIZE(mul32)
2724 
2725 #elif defined(__i386)
2726 
2727         ENTRY(mul32)
2728         movl    8(%esp), %eax
2729         movl    4(%esp), %ecx
2730         mull    %ecx
2731         ret
2732         SET_SIZE(mul32)
2733 
2734 #endif  /* __i386 */
2735 #endif  /* __lint */
2736 
2737 #if defined(notused)
2738 #if defined(__lint)
2739 /* ARGSUSED */
2740 void
2741 load_pte64(uint64_t *pte, uint64_t pte_value)
2742 {}
2743 #else   /* __lint */
2744         .globl load_pte64
2745 load_pte64:
2746         movl    4(%esp), %eax
2747         movl    8(%esp), %ecx
2748         movl    12(%esp), %edx
2749         movl    %edx, 4(%eax)
2750         movl    %ecx, (%eax)
2751         ret
2752 #endif  /* __lint */
2753 #endif  /* notused */
2754 
2755 #if defined(__lint)
2756 
2757 /*ARGSUSED*/
2758 void
2759 scan_memory(caddr_t addr, size_t size)
2760 {}
2761 
2762 #else   /* __lint */
2763 
2764 #if defined(__amd64)
2765 
2766         ENTRY(scan_memory)
2767         shrq    $3, %rsi        /* convert %rsi from byte to quadword count */
2768         jz      .scanm_done
2769         movq    %rsi, %rcx      /* move count into rep control register */
2770         movq    %rdi, %rsi      /* move addr into lodsq control reg. */
2771         rep lodsq               /* scan the memory range */
2772 .scanm_done:
2773         rep;    ret     /* use 2 byte return instruction when branch target */
2774                         /* AMD Software Optimization Guide - Section 6.2 */
2775         SET_SIZE(scan_memory)
2776 
2777 #elif defined(__i386)
2778 
2779         ENTRY(scan_memory)
2780         pushl   %ecx
2781         pushl   %esi
2782         movl    16(%esp), %ecx  /* move 2nd arg into rep control register */
2783         shrl    $2, %ecx        /* convert from byte count to word count */
2784         jz      .scanm_done
2785         movl    12(%esp), %esi  /* move 1st arg into lodsw control register */
2786         .byte   0xf3            /* rep prefix.  lame assembler.  sigh. */
2787         lodsl
2788 .scanm_done:
2789         popl    %esi
2790         popl    %ecx
2791         ret
2792         SET_SIZE(scan_memory)
2793 
2794 #endif  /* __i386 */
2795 #endif  /* __lint */
2796 
2797 
2798 #if defined(__lint)
2799 
2800 /*ARGSUSED */
2801 int
2802 lowbit(ulong_t i)
2803 { return (0); }
2804 
2805 #else   /* __lint */
2806 
2807 #if defined(__amd64)
2808 
2809         ENTRY(lowbit)
2810         movl    $-1, %eax
2811         bsfq    %rdi, %rdi
2812         cmovnz  %edi, %eax
2813         incl    %eax
2814         ret
2815         SET_SIZE(lowbit)
2816 
2817 #elif defined(__i386)
2818 
2819         ENTRY(lowbit)
2820         bsfl    4(%esp), %eax
2821         jz      0f
2822         incl    %eax
2823         ret
2824 0:
2825         xorl    %eax, %eax
2826         ret
2827         SET_SIZE(lowbit)
2828 
2829 #endif  /* __i386 */
2830 #endif  /* __lint */
2831 
2832 #if defined(__lint)
2833 
2834 /*ARGSUSED*/
2835 int
2836 highbit(ulong_t i)
2837 { return (0); }
2838 
2839 /*ARGSUSED*/
2840 int
2841 highbit64(uint64_t i)
2842 { return (0); }
2843 
2844 #else   /* __lint */
2845 
2846 #if defined(__amd64)
2847 
2848         ENTRY(highbit)
2849         ALTENTRY(highbit64)
2850         movl    $-1, %eax
2851         bsrq    %rdi, %rdi
2852         cmovnz  %edi, %eax
2853         incl    %eax
2854         ret
2855         SET_SIZE(highbit64)
2856         SET_SIZE(highbit)
2857 
2858 #elif defined(__i386)
2859 
2860         ENTRY(highbit)
2861         bsrl    4(%esp), %eax
2862         jz      0f
2863         incl    %eax
2864         ret
2865 0:
2866         xorl    %eax, %eax
2867         ret    
2868         SET_SIZE(highbit)
2869 
2870         ENTRY(highbit64)
2871         bsrl    8(%esp), %eax
2872         jz      highbit
2873         addl    $33, %eax
2874         ret
2875         SET_SIZE(highbit64)
2876 
2877 #endif  /* __i386 */
2878 #endif  /* __lint */
2879 
2880 #if defined(__lint)
2881 
2882 /*ARGSUSED*/
2883 uint64_t
2884 rdmsr(uint_t r)
2885 { return (0); }
2886 
2887 /*ARGSUSED*/
2888 void
2889 wrmsr(uint_t r, const uint64_t val)
2890 {}
2891 
2892 /*ARGSUSED*/
2893 uint64_t
2894 xrdmsr(uint_t r)
2895 { return (0); }
2896 
2897 /*ARGSUSED*/
2898 void
2899 xwrmsr(uint_t r, const uint64_t val)
2900 {}
2901 
2902 void
2903 invalidate_cache(void)
2904 {}
2905 
2906 /*ARGSUSED*/
2907 uint64_t
2908 get_xcr(uint_t r)
2909 { return (0); }
2910 
2911 /*ARGSUSED*/
2912 void
2913 set_xcr(uint_t r, const uint64_t val)
2914 {}
2915 
2916 #else  /* __lint */
2917 
2918 #define XMSR_ACCESS_VAL         $0x9c5a203a
2919 
2920 #if defined(__amd64)
2921         
2922         ENTRY(rdmsr)
2923         movl    %edi, %ecx
2924         rdmsr
2925         shlq    $32, %rdx
2926         orq     %rdx, %rax
2927         ret
2928         SET_SIZE(rdmsr)
2929 
2930         ENTRY(wrmsr)
2931         movq    %rsi, %rdx
2932         shrq    $32, %rdx
2933         movl    %esi, %eax
2934         movl    %edi, %ecx
2935         wrmsr
2936         ret
2937         SET_SIZE(wrmsr)
2938 
2939         ENTRY(xrdmsr)
2940         pushq   %rbp
2941         movq    %rsp, %rbp
2942         movl    %edi, %ecx
2943         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
2944         rdmsr
2945         shlq    $32, %rdx
2946         orq     %rdx, %rax
2947         leave
2948         ret
2949         SET_SIZE(xrdmsr)
2950 
2951         ENTRY(xwrmsr)
2952         pushq   %rbp
2953         movq    %rsp, %rbp
2954         movl    %edi, %ecx
2955         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
2956         movq    %rsi, %rdx
2957         shrq    $32, %rdx
2958         movl    %esi, %eax
2959         wrmsr
2960         leave
2961         ret
2962         SET_SIZE(xwrmsr)
2963 
2964         ENTRY(get_xcr)
2965         movl    %edi, %ecx
2966         #xgetbv
2967         .byte   0x0f,0x01,0xd0
2968         shlq    $32, %rdx
2969         orq     %rdx, %rax
2970         ret
2971         SET_SIZE(get_xcr)
2972 
2973         ENTRY(set_xcr)
2974         movq    %rsi, %rdx
2975         shrq    $32, %rdx
2976         movl    %esi, %eax
2977         movl    %edi, %ecx
2978         #xsetbv
2979         .byte   0x0f,0x01,0xd1
2980         ret
2981         SET_SIZE(set_xcr)
2982 
2983 #elif defined(__i386)
2984 
2985         ENTRY(rdmsr)
2986         movl    4(%esp), %ecx
2987         rdmsr
2988         ret
2989         SET_SIZE(rdmsr)
2990 
2991         ENTRY(wrmsr)
2992         movl    4(%esp), %ecx
2993         movl    8(%esp), %eax
2994         movl    12(%esp), %edx 
2995         wrmsr
2996         ret
2997         SET_SIZE(wrmsr)
2998 
2999         ENTRY(xrdmsr)
3000         pushl   %ebp
3001         movl    %esp, %ebp
3002         movl    8(%esp), %ecx
3003         pushl   %edi
3004         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
3005         rdmsr
3006         popl    %edi
3007         leave
3008         ret
3009         SET_SIZE(xrdmsr)
3010 
3011         ENTRY(xwrmsr)
3012         pushl   %ebp
3013         movl    %esp, %ebp
3014         movl    8(%esp), %ecx
3015         movl    12(%esp), %eax
3016         movl    16(%esp), %edx 
3017         pushl   %edi
3018         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
3019         wrmsr
3020         popl    %edi
3021         leave
3022         ret
3023         SET_SIZE(xwrmsr)
3024 
3025         ENTRY(get_xcr)
3026         movl    4(%esp), %ecx
3027         #xgetbv
3028         .byte   0x0f,0x01,0xd0
3029         ret
3030         SET_SIZE(get_xcr)
3031 
3032         ENTRY(set_xcr)
3033         movl    4(%esp), %ecx
3034         movl    8(%esp), %eax
3035         movl    12(%esp), %edx
3036         #xsetbv
3037         .byte   0x0f,0x01,0xd1
3038         ret
3039         SET_SIZE(set_xcr)
3040 
3041 #endif  /* __i386 */
3042 
3043         ENTRY(invalidate_cache)
3044         wbinvd
3045         ret
3046         SET_SIZE(invalidate_cache)
3047 
3048 #endif  /* __lint */
3049 
3050 #if defined(__lint)
3051 
3052 /*ARGSUSED*/
3053 void
3054 getcregs(struct cregs *crp)
3055 {}
3056 
3057 #else   /* __lint */
3058 
3059 #if defined(__amd64)
3060 
3061         ENTRY_NP(getcregs)
3062 #if defined(__xpv)
3063         /*
3064          * Only a few of the hardware control registers or descriptor tables
3065          * are directly accessible to us, so just zero the structure.
3066          *
3067          * XXPV Perhaps it would be helpful for the hypervisor to return
3068          *      virtualized versions of these for post-mortem use.
3069          *      (Need to reevaluate - perhaps it already does!)
3070          */
3071         pushq   %rdi            /* save *crp */
3072         movq    $CREGSZ, %rsi
3073         call    bzero
3074         popq    %rdi
3075 
3076         /*
3077          * Dump what limited information we can
3078          */
3079         movq    %cr0, %rax
3080         movq    %rax, CREG_CR0(%rdi)    /* cr0 */
3081         movq    %cr2, %rax
3082         movq    %rax, CREG_CR2(%rdi)    /* cr2 */
3083         movq    %cr3, %rax
3084         movq    %rax, CREG_CR3(%rdi)    /* cr3 */
3085         movq    %cr4, %rax
3086         movq    %rax, CREG_CR4(%rdi)    /* cr4 */
3087 
3088 #else   /* __xpv */
3089 
3090 #define GETMSR(r, off, d)       \
3091         movl    $r, %ecx;       \
3092         rdmsr;                  \
3093         movl    %eax, off(d);   \
3094         movl    %edx, off+4(d)
3095 
3096         xorl    %eax, %eax
3097         movq    %rax, CREG_GDT+8(%rdi)
3098         sgdt    CREG_GDT(%rdi)          /* 10 bytes */
3099         movq    %rax, CREG_IDT+8(%rdi)
3100         sidt    CREG_IDT(%rdi)          /* 10 bytes */
3101         movq    %rax, CREG_LDT(%rdi)
3102         sldt    CREG_LDT(%rdi)          /* 2 bytes */
3103         movq    %rax, CREG_TASKR(%rdi)
3104         str     CREG_TASKR(%rdi)        /* 2 bytes */
3105         movq    %cr0, %rax
3106         movq    %rax, CREG_CR0(%rdi)    /* cr0 */
3107         movq    %cr2, %rax
3108         movq    %rax, CREG_CR2(%rdi)    /* cr2 */
3109         movq    %cr3, %rax
3110         movq    %rax, CREG_CR3(%rdi)    /* cr3 */
3111         movq    %cr4, %rax
3112         movq    %rax, CREG_CR4(%rdi)    /* cr4 */
3113         movq    %cr8, %rax
3114         movq    %rax, CREG_CR8(%rdi)    /* cr8 */
3115         GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3116         GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3117 #endif  /* __xpv */
3118         ret
3119         SET_SIZE(getcregs)
3120 
3121 #undef GETMSR
3122 
3123 #elif defined(__i386)
3124 
3125         ENTRY_NP(getcregs)
3126 #if defined(__xpv)
3127         /*
3128          * Only a few of the hardware control registers or descriptor tables
3129          * are directly accessible to us, so just zero the structure.
3130          *
3131          * XXPV Perhaps it would be helpful for the hypervisor to return
3132          *      virtualized versions of these for post-mortem use.
3133          *      (Need to reevaluate - perhaps it already does!)
3134          */
3135         movl    4(%esp), %edx
3136         pushl   $CREGSZ
3137         pushl   %edx
3138         call    bzero
3139         addl    $8, %esp
3140         movl    4(%esp), %edx
3141 
3142         /*
3143          * Dump what limited information we can
3144          */
3145         movl    %cr0, %eax
3146         movl    %eax, CREG_CR0(%edx)    /* cr0 */
3147         movl    %cr2, %eax
3148         movl    %eax, CREG_CR2(%edx)    /* cr2 */
3149         movl    %cr3, %eax
3150         movl    %eax, CREG_CR3(%edx)    /* cr3 */
3151         movl    %cr4, %eax
3152         movl    %eax, CREG_CR4(%edx)    /* cr4 */
3153 
3154 #else   /* __xpv */
3155 
3156         movl    4(%esp), %edx
3157         movw    $0, CREG_GDT+6(%edx)
3158         movw    $0, CREG_IDT+6(%edx)
3159         sgdt    CREG_GDT(%edx)          /* gdt */
3160         sidt    CREG_IDT(%edx)          /* idt */
3161         sldt    CREG_LDT(%edx)          /* ldt */
3162         str     CREG_TASKR(%edx)        /* task */
3163         movl    %cr0, %eax
3164         movl    %eax, CREG_CR0(%edx)    /* cr0 */
3165         movl    %cr2, %eax
3166         movl    %eax, CREG_CR2(%edx)    /* cr2 */
3167         movl    %cr3, %eax
3168         movl    %eax, CREG_CR3(%edx)    /* cr3 */
3169         bt      $X86FSET_LARGEPAGE, x86_featureset
3170         jnc     .nocr4
3171         movl    %cr4, %eax
3172         movl    %eax, CREG_CR4(%edx)    /* cr4 */
3173         jmp     .skip
3174 .nocr4:
3175         movl    $0, CREG_CR4(%edx)
3176 .skip:
3177 #endif
3178         ret
3179         SET_SIZE(getcregs)
3180 
3181 #endif  /* __i386 */
3182 #endif  /* __lint */
3183 
3184 
3185 /*
3186  * A panic trigger is a word which is updated atomically and can only be set
3187  * once.  We atomically store 0xDEFACEDD and load the old value.  If the
3188  * previous value was 0, we succeed and return 1; otherwise return 0.
3189  * This allows a partially corrupt trigger to still trigger correctly.  DTrace
3190  * has its own version of this function to allow it to panic correctly from
3191  * probe context.
3192  */
3193 #if defined(__lint)
3194 
3195 /*ARGSUSED*/
3196 int
3197 panic_trigger(int *tp)
3198 { return (0); }
3199 
3200 /*ARGSUSED*/
3201 int
3202 dtrace_panic_trigger(int *tp)
3203 { return (0); }
3204 
3205 #else   /* __lint */
3206 
3207 #if defined(__amd64)
3208 
3209         ENTRY_NP(panic_trigger)
3210         xorl    %eax, %eax
3211         movl    $0xdefacedd, %edx
3212         lock
3213           xchgl %edx, (%rdi)
3214         cmpl    $0, %edx
3215         je      0f 
3216         movl    $0, %eax
3217         ret
3218 0:      movl    $1, %eax
3219         ret
3220         SET_SIZE(panic_trigger)
3221         
3222         ENTRY_NP(dtrace_panic_trigger)
3223         xorl    %eax, %eax
3224         movl    $0xdefacedd, %edx
3225         lock
3226           xchgl %edx, (%rdi)
3227         cmpl    $0, %edx
3228         je      0f
3229         movl    $0, %eax
3230         ret
3231 0:      movl    $1, %eax
3232         ret
3233         SET_SIZE(dtrace_panic_trigger)
3234 
3235 #elif defined(__i386)
3236 
3237         ENTRY_NP(panic_trigger)
3238         movl    4(%esp), %edx           / %edx = address of trigger
3239         movl    $0xdefacedd, %eax       / %eax = 0xdefacedd
3240         lock                            / assert lock
3241         xchgl %eax, (%edx)              / exchange %eax and the trigger
3242         cmpl    $0, %eax                / if (%eax == 0x0)
3243         je      0f                      /   return (1);
3244         movl    $0, %eax                / else
3245         ret                             /   return (0);
3246 0:      movl    $1, %eax
3247         ret
3248         SET_SIZE(panic_trigger)
3249 
3250         ENTRY_NP(dtrace_panic_trigger)
3251         movl    4(%esp), %edx           / %edx = address of trigger
3252         movl    $0xdefacedd, %eax       / %eax = 0xdefacedd
3253         lock                            / assert lock
3254         xchgl %eax, (%edx)              / exchange %eax and the trigger
3255         cmpl    $0, %eax                / if (%eax == 0x0)
3256         je      0f                      /   return (1);
3257         movl    $0, %eax                / else
3258         ret                             /   return (0);
3259 0:      movl    $1, %eax
3260         ret
3261         SET_SIZE(dtrace_panic_trigger)
3262 
3263 #endif  /* __i386 */
3264 #endif  /* __lint */
3265 
3266 /*
3267  * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3268  * into the panic code implemented in panicsys().  vpanic() is responsible
3269  * for passing through the format string and arguments, and constructing a
3270  * regs structure on the stack into which it saves the current register
3271  * values.  If we are not dying due to a fatal trap, these registers will
3272  * then be preserved in panicbuf as the current processor state.  Before
3273  * invoking panicsys(), vpanic() activates the first panic trigger (see
3274  * common/os/panic.c) and switches to the panic_stack if successful.  Note that
3275  * DTrace takes a slightly different panic path if it must panic from probe
3276  * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
3277  * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3278  * branches back into vpanic().
3279  */
3280 #if defined(__lint)
3281 
3282 /*ARGSUSED*/
3283 void
3284 vpanic(const char *format, va_list alist)
3285 {}
3286 
3287 /*ARGSUSED*/
3288 void
3289 dtrace_vpanic(const char *format, va_list alist)
3290 {}
3291 
3292 #else   /* __lint */
3293 
3294 #if defined(__amd64)
3295 
3296         ENTRY_NP(vpanic)                        /* Initial stack layout: */
3297         
3298         pushq   %rbp                            /* | %rip |     0x60    */
3299         movq    %rsp, %rbp                      /* | %rbp |     0x58    */
3300         pushfq                                  /* | rfl  |     0x50    */
3301         pushq   %r11                            /* | %r11 |     0x48    */
3302         pushq   %r10                            /* | %r10 |     0x40    */
3303         pushq   %rbx                            /* | %rbx |     0x38    */
3304         pushq   %rax                            /* | %rax |     0x30    */
3305         pushq   %r9                             /* | %r9  |     0x28    */
3306         pushq   %r8                             /* | %r8  |     0x20    */
3307         pushq   %rcx                            /* | %rcx |     0x18    */
3308         pushq   %rdx                            /* | %rdx |     0x10    */
3309         pushq   %rsi                            /* | %rsi |     0x8 alist */
3310         pushq   %rdi                            /* | %rdi |     0x0 format */
3311 
3312         movq    %rsp, %rbx                      /* %rbx = current %rsp */
3313 
3314         leaq    panic_quiesce(%rip), %rdi       /* %rdi = &panic_quiesce */
3315         call    panic_trigger                   /* %eax = panic_trigger() */
3316 
3317 vpanic_common:
3318         /*
3319          * The panic_trigger result is in %eax from the call above, and
3320          * dtrace_panic places it in %eax before branching here.
3321          * The rdmsr instructions that follow below will clobber %eax so
3322          * we stash the panic_trigger result in %r11d.
3323          */
3324         movl    %eax, %r11d
3325         cmpl    $0, %r11d
3326         je      0f
3327 
3328         /*
3329          * If panic_trigger() was successful, we are the first to initiate a
3330          * panic: we now switch to the reserved panic_stack before continuing.
3331          */
3332         leaq    panic_stack(%rip), %rsp
3333         addq    $PANICSTKSIZE, %rsp
3334 0:      subq    $REGSIZE, %rsp
3335         /*
3336          * Now that we've got everything set up, store the register values as
3337          * they were when we entered vpanic() to the designated location in
3338          * the regs structure we allocated on the stack.
3339          */
3340         movq    0x0(%rbx), %rcx
3341         movq    %rcx, REGOFF_RDI(%rsp)
3342         movq    0x8(%rbx), %rcx
3343         movq    %rcx, REGOFF_RSI(%rsp)
3344         movq    0x10(%rbx), %rcx
3345         movq    %rcx, REGOFF_RDX(%rsp)
3346         movq    0x18(%rbx), %rcx
3347         movq    %rcx, REGOFF_RCX(%rsp)
3348         movq    0x20(%rbx), %rcx
3349 
3350         movq    %rcx, REGOFF_R8(%rsp)
3351         movq    0x28(%rbx), %rcx
3352         movq    %rcx, REGOFF_R9(%rsp)
3353         movq    0x30(%rbx), %rcx
3354         movq    %rcx, REGOFF_RAX(%rsp)
3355         movq    0x38(%rbx), %rcx
3356         movq    %rcx, REGOFF_RBX(%rsp)
3357         movq    0x58(%rbx), %rcx
3358 
3359         movq    %rcx, REGOFF_RBP(%rsp)
3360         movq    0x40(%rbx), %rcx
3361         movq    %rcx, REGOFF_R10(%rsp)
3362         movq    0x48(%rbx), %rcx
3363         movq    %rcx, REGOFF_R11(%rsp)
3364         movq    %r12, REGOFF_R12(%rsp)
3365 
3366         movq    %r13, REGOFF_R13(%rsp)
3367         movq    %r14, REGOFF_R14(%rsp)
3368         movq    %r15, REGOFF_R15(%rsp)
3369 
3370         xorl    %ecx, %ecx
3371         movw    %ds, %cx
3372         movq    %rcx, REGOFF_DS(%rsp)
3373         movw    %es, %cx
3374         movq    %rcx, REGOFF_ES(%rsp)
3375         movw    %fs, %cx
3376         movq    %rcx, REGOFF_FS(%rsp)
3377         movw    %gs, %cx
3378         movq    %rcx, REGOFF_GS(%rsp)
3379 
3380         movq    $0, REGOFF_TRAPNO(%rsp)
3381 
3382         movq    $0, REGOFF_ERR(%rsp)
3383         leaq    vpanic(%rip), %rcx
3384         movq    %rcx, REGOFF_RIP(%rsp)
3385         movw    %cs, %cx
3386         movzwq  %cx, %rcx
3387         movq    %rcx, REGOFF_CS(%rsp)
3388         movq    0x50(%rbx), %rcx
3389         movq    %rcx, REGOFF_RFL(%rsp)
3390         movq    %rbx, %rcx
3391         addq    $0x60, %rcx
3392         movq    %rcx, REGOFF_RSP(%rsp)
3393         movw    %ss, %cx
3394         movzwq  %cx, %rcx
3395         movq    %rcx, REGOFF_SS(%rsp)
3396 
3397         /*
3398          * panicsys(format, alist, rp, on_panic_stack) 
3399          */     
3400         movq    REGOFF_RDI(%rsp), %rdi          /* format */
3401         movq    REGOFF_RSI(%rsp), %rsi          /* alist */
3402         movq    %rsp, %rdx                      /* struct regs */
3403         movl    %r11d, %ecx                     /* on_panic_stack */
3404         call    panicsys
3405         addq    $REGSIZE, %rsp
3406         popq    %rdi
3407         popq    %rsi
3408         popq    %rdx
3409         popq    %rcx
3410         popq    %r8
3411         popq    %r9
3412         popq    %rax
3413         popq    %rbx
3414         popq    %r10
3415         popq    %r11
3416         popfq
3417         leave
3418         ret
3419         SET_SIZE(vpanic)
3420 
3421         ENTRY_NP(dtrace_vpanic)                 /* Initial stack layout: */
3422 
3423         pushq   %rbp                            /* | %rip |     0x60    */
3424         movq    %rsp, %rbp                      /* | %rbp |     0x58    */
3425         pushfq                                  /* | rfl  |     0x50    */
3426         pushq   %r11                            /* | %r11 |     0x48    */
3427         pushq   %r10                            /* | %r10 |     0x40    */
3428         pushq   %rbx                            /* | %rbx |     0x38    */
3429         pushq   %rax                            /* | %rax |     0x30    */
3430         pushq   %r9                             /* | %r9  |     0x28    */
3431         pushq   %r8                             /* | %r8  |     0x20    */
3432         pushq   %rcx                            /* | %rcx |     0x18    */
3433         pushq   %rdx                            /* | %rdx |     0x10    */
3434         pushq   %rsi                            /* | %rsi |     0x8 alist */
3435         pushq   %rdi                            /* | %rdi |     0x0 format */
3436 
3437         movq    %rsp, %rbx                      /* %rbx = current %rsp */
3438 
3439         leaq    panic_quiesce(%rip), %rdi       /* %rdi = &panic_quiesce */
3440         call    dtrace_panic_trigger    /* %eax = dtrace_panic_trigger() */
3441         jmp     vpanic_common
3442 
3443         SET_SIZE(dtrace_vpanic)
3444 
3445 #elif defined(__i386)
3446 
3447         ENTRY_NP(vpanic)                        / Initial stack layout:
3448 
3449         pushl   %ebp                            / | %eip | 20
3450         movl    %esp, %ebp                      / | %ebp | 16
3451         pushl   %eax                            / | %eax | 12
3452         pushl   %ebx                            / | %ebx |  8
3453         pushl   %ecx                            / | %ecx |  4
3454         pushl   %edx                            / | %edx |  0
3455 
3456         movl    %esp, %ebx                      / %ebx = current stack pointer
3457 
3458         lea     panic_quiesce, %eax             / %eax = &panic_quiesce
3459         pushl   %eax                            / push &panic_quiesce
3460         call    panic_trigger                   / %eax = panic_trigger()
3461         addl    $4, %esp                        / reset stack pointer
3462 
3463 vpanic_common:
3464         cmpl    $0, %eax                        / if (%eax == 0)
3465         je      0f                              /   goto 0f;
3466 
3467         /*
3468          * If panic_trigger() was successful, we are the first to initiate a
3469          * panic: we now switch to the reserved panic_stack before continuing.
3470          */
3471         lea     panic_stack, %esp               / %esp  = panic_stack
3472         addl    $PANICSTKSIZE, %esp             / %esp += PANICSTKSIZE
3473 
3474 0:      subl    $REGSIZE, %esp                  / allocate struct regs
3475 
3476         /*
3477          * Now that we've got everything set up, store the register values as
3478          * they were when we entered vpanic() to the designated location in
3479          * the regs structure we allocated on the stack. 
3480          */
3481 #if !defined(__GNUC_AS__)
3482         movw    %gs, %edx
3483         movl    %edx, REGOFF_GS(%esp)
3484         movw    %fs, %edx
3485         movl    %edx, REGOFF_FS(%esp)
3486         movw    %es, %edx
3487         movl    %edx, REGOFF_ES(%esp)
3488         movw    %ds, %edx
3489         movl    %edx, REGOFF_DS(%esp)
3490 #else   /* __GNUC_AS__ */
3491         mov     %gs, %edx
3492         mov     %edx, REGOFF_GS(%esp)
3493         mov     %fs, %edx
3494         mov     %edx, REGOFF_FS(%esp)
3495         mov     %es, %edx
3496         mov     %edx, REGOFF_ES(%esp)
3497         mov     %ds, %edx
3498         mov     %edx, REGOFF_DS(%esp)
3499 #endif  /* __GNUC_AS__ */
3500         movl    %edi, REGOFF_EDI(%esp)
3501         movl    %esi, REGOFF_ESI(%esp)
3502         movl    16(%ebx), %ecx
3503         movl    %ecx, REGOFF_EBP(%esp)
3504         movl    %ebx, %ecx
3505         addl    $20, %ecx
3506         movl    %ecx, REGOFF_ESP(%esp)
3507         movl    8(%ebx), %ecx
3508         movl    %ecx, REGOFF_EBX(%esp)
3509         movl    0(%ebx), %ecx
3510         movl    %ecx, REGOFF_EDX(%esp)
3511         movl    4(%ebx), %ecx
3512         movl    %ecx, REGOFF_ECX(%esp)
3513         movl    12(%ebx), %ecx
3514         movl    %ecx, REGOFF_EAX(%esp)
3515         movl    $0, REGOFF_TRAPNO(%esp)
3516         movl    $0, REGOFF_ERR(%esp)
3517         lea     vpanic, %ecx
3518         movl    %ecx, REGOFF_EIP(%esp)
3519 #if !defined(__GNUC_AS__)
3520         movw    %cs, %edx
3521 #else   /* __GNUC_AS__ */
3522         mov     %cs, %edx
3523 #endif  /* __GNUC_AS__ */
3524         movl    %edx, REGOFF_CS(%esp)
3525         pushfl
3526         popl    %ecx
3527 #if defined(__xpv)
3528         /*
3529          * Synthesize the PS_IE bit from the event mask bit
3530          */
3531         CURTHREAD(%edx)
3532         KPREEMPT_DISABLE(%edx)
3533         EVENT_MASK_TO_IE(%edx, %ecx)
3534         CURTHREAD(%edx)
3535         KPREEMPT_ENABLE_NOKP(%edx)
3536 #endif
3537         movl    %ecx, REGOFF_EFL(%esp)
3538         movl    $0, REGOFF_UESP(%esp)
3539 #if !defined(__GNUC_AS__)
3540         movw    %ss, %edx
3541 #else   /* __GNUC_AS__ */
3542         mov     %ss, %edx
3543 #endif  /* __GNUC_AS__ */
3544         movl    %edx, REGOFF_SS(%esp)
3545 
3546         movl    %esp, %ecx                      / %ecx = &regs
3547         pushl   %eax                            / push on_panic_stack
3548         pushl   %ecx                            / push &regs
3549         movl    12(%ebp), %ecx                  / %ecx = alist
3550         pushl   %ecx                            / push alist
3551         movl    8(%ebp), %ecx                   / %ecx = format
3552         pushl   %ecx                            / push format
3553         call    panicsys                        / panicsys();
3554         addl    $16, %esp                       / pop arguments
3555 
3556         addl    $REGSIZE, %esp
3557         popl    %edx
3558         popl    %ecx
3559         popl    %ebx
3560         popl    %eax
3561         leave
3562         ret
3563         SET_SIZE(vpanic)
3564 
3565         ENTRY_NP(dtrace_vpanic)                 / Initial stack layout:
3566 
3567         pushl   %ebp                            / | %eip | 20
3568         movl    %esp, %ebp                      / | %ebp | 16
3569         pushl   %eax                            / | %eax | 12
3570         pushl   %ebx                            / | %ebx |  8
3571         pushl   %ecx                            / | %ecx |  4
3572         pushl   %edx                            / | %edx |  0
3573 
3574         movl    %esp, %ebx                      / %ebx = current stack pointer
3575 
3576         lea     panic_quiesce, %eax             / %eax = &panic_quiesce
3577         pushl   %eax                            / push &panic_quiesce
3578         call    dtrace_panic_trigger            / %eax = dtrace_panic_trigger()
3579         addl    $4, %esp                        / reset stack pointer
3580         jmp     vpanic_common                   / jump back to common code
3581 
3582         SET_SIZE(dtrace_vpanic)
3583 
3584 #endif  /* __i386 */
3585 #endif  /* __lint */
3586 
3587 #if defined(__lint)
3588 
3589 void
3590 hres_tick(void)
3591 {}
3592 
3593 int64_t timedelta;
3594 hrtime_t hrtime_base;
3595 
3596 #else   /* __lint */
3597 
3598         DGDEF3(timedelta, 8, 8)
3599         .long   0, 0
3600 
3601         /*
3602          * initialized to a non zero value to make pc_gethrtime()
3603          * work correctly even before clock is initialized
3604          */
3605         DGDEF3(hrtime_base, 8, 8)
3606         .long   _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3607 
3608         DGDEF3(adj_shift, 4, 4)
3609         .long   ADJ_SHIFT
3610 
3611 #if defined(__amd64)
3612 
3613         ENTRY_NP(hres_tick)
3614         pushq   %rbp
3615         movq    %rsp, %rbp
3616 
3617         /*
3618          * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3619          * hres_last_tick can only be modified while holding CLOCK_LOCK).
3620          * At worst, performing this now instead of under CLOCK_LOCK may
3621          * introduce some jitter in pc_gethrestime().
3622          */
3623         call    *gethrtimef(%rip)
3624         movq    %rax, %r8
3625 
3626         leaq    hres_lock(%rip), %rax
3627         movb    $-1, %dl
3628 .CL1:
3629         xchgb   %dl, (%rax)
3630         testb   %dl, %dl
3631         jz      .CL3                    /* got it */
3632 .CL2:
3633         cmpb    $0, (%rax)              /* possible to get lock? */
3634         pause
3635         jne     .CL2
3636         jmp     .CL1                    /* yes, try again */
3637 .CL3:
3638         /*
3639          * compute the interval since last time hres_tick was called
3640          * and adjust hrtime_base and hrestime accordingly
3641          * hrtime_base is an 8 byte value (in nsec), hrestime is
3642          * a timestruc_t (sec, nsec)
3643          */
3644         leaq    hres_last_tick(%rip), %rax
3645         movq    %r8, %r11
3646         subq    (%rax), %r8
3647         addq    %r8, hrtime_base(%rip)  /* add interval to hrtime_base */
3648         addq    %r8, hrestime+8(%rip)   /* add interval to hrestime.tv_nsec */
3649         /*
3650          * Now that we have CLOCK_LOCK, we can update hres_last_tick
3651          */     
3652         movq    %r11, (%rax)    
3653 
3654         call    __adj_hrestime
3655 
3656         /*
3657          * release the hres_lock
3658          */
3659         incl    hres_lock(%rip)
3660         leave
3661         ret
3662         SET_SIZE(hres_tick)
3663         
3664 #elif defined(__i386)
3665 
3666         ENTRY_NP(hres_tick)
3667         pushl   %ebp
3668         movl    %esp, %ebp
3669         pushl   %esi
3670         pushl   %ebx
3671 
3672         /*
3673          * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3674          * hres_last_tick can only be modified while holding CLOCK_LOCK).
3675          * At worst, performing this now instead of under CLOCK_LOCK may
3676          * introduce some jitter in pc_gethrestime().
3677          */
3678         call    *gethrtimef
3679         movl    %eax, %ebx
3680         movl    %edx, %esi
3681 
3682         movl    $hres_lock, %eax
3683         movl    $-1, %edx
3684 .CL1:
3685         xchgb   %dl, (%eax)
3686         testb   %dl, %dl
3687         jz      .CL3                    / got it
3688 .CL2:
3689         cmpb    $0, (%eax)              / possible to get lock?
3690         pause
3691         jne     .CL2
3692         jmp     .CL1                    / yes, try again
3693 .CL3:
3694         /*
3695          * compute the interval since last time hres_tick was called
3696          * and adjust hrtime_base and hrestime accordingly
3697          * hrtime_base is an 8 byte value (in nsec), hrestime is
3698          * timestruc_t (sec, nsec)
3699          */
3700 
3701         lea     hres_last_tick, %eax
3702 
3703         movl    %ebx, %edx
3704         movl    %esi, %ecx
3705 
3706         subl    (%eax), %edx
3707         sbbl    4(%eax), %ecx
3708 
3709         addl    %edx, hrtime_base       / add interval to hrtime_base
3710         adcl    %ecx, hrtime_base+4
3711 
3712         addl    %edx, hrestime+4        / add interval to hrestime.tv_nsec
3713 
3714         /
3715         / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3716         /
3717         movl    %ebx, (%eax)
3718         movl    %esi,  4(%eax)
3719 
3720         / get hrestime at this moment. used as base for pc_gethrestime
3721         /
3722         / Apply adjustment, if any
3723         /
3724         / #define HRES_ADJ      (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3725         / (max_hres_adj)
3726         /
3727         / void
3728         / adj_hrestime()
3729         / {
3730         /       long long adj;
3731         /
3732         /       if (hrestime_adj == 0)
3733         /               adj = 0;
3734         /       else if (hrestime_adj > 0) {
3735         /               if (hrestime_adj < HRES_ADJ)
3736         /                       adj = hrestime_adj;
3737         /               else
3738         /                       adj = HRES_ADJ;
3739         /       }
3740         /       else {
3741         /               if (hrestime_adj < -(HRES_ADJ))
3742         /                       adj = -(HRES_ADJ);
3743         /               else
3744         /                       adj = hrestime_adj;
3745         /       }
3746         /
3747         /       timedelta -= adj;
3748         /       hrestime_adj = timedelta;
3749         /       hrestime.tv_nsec += adj;
3750         /
3751         /       while (hrestime.tv_nsec >= NANOSEC) {
3752         /               one_sec++;
3753         /               hrestime.tv_sec++;
3754         /               hrestime.tv_nsec -= NANOSEC;
3755         /       }
3756         / }
3757 __adj_hrestime:
3758         movl    hrestime_adj, %esi      / if (hrestime_adj == 0)
3759         movl    hrestime_adj+4, %edx
3760         andl    %esi, %esi
3761         jne     .CL4                    / no
3762         andl    %edx, %edx
3763         jne     .CL4                    / no
3764         subl    %ecx, %ecx              / yes, adj = 0;
3765         subl    %edx, %edx
3766         jmp     .CL5
3767 .CL4:
3768         subl    %ecx, %ecx
3769         subl    %eax, %eax
3770         subl    %esi, %ecx
3771         sbbl    %edx, %eax
3772         andl    %eax, %eax              / if (hrestime_adj > 0)
3773         jge     .CL6
3774 
3775         / In the following comments, HRES_ADJ is used, while in the code
3776         / max_hres_adj is used.
3777         /
3778         / The test for "hrestime_adj < HRES_ADJ" is complicated because
3779         / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits.  We rely
3780         / on the logical equivalence of:
3781         /
3782         /       !(hrestime_adj < HRES_ADJ)
3783         /
3784         / and the two step sequence:
3785         /
3786         /       (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3787         /
3788         / which computes whether or not the least significant 32-bits
3789         / of hrestime_adj is greater than HRES_ADJ, followed by:
3790         /
3791         /       Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3792         /
3793         / which generates a carry whenever step 1 is true or the most
3794         / significant long of the longlong hrestime_adj is non-zero.
3795 
3796         movl    max_hres_adj, %ecx      / hrestime_adj is positive
3797         subl    %esi, %ecx
3798         movl    %edx, %eax
3799         adcl    $-1, %eax
3800         jnc     .CL7
3801         movl    max_hres_adj, %ecx      / adj = HRES_ADJ;
3802         subl    %edx, %edx
3803         jmp     .CL5
3804 
3805         / The following computation is similar to the one above.
3806         /
3807         / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3808         / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits.  We rely
3809         / on the logical equivalence of:
3810         /
3811         /       (hrestime_adj > -HRES_ADJ)
3812         /
3813         / and the two step sequence:
3814         /
3815         /       (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3816         /
3817         / which means the least significant 32-bits of hrestime_adj is
3818         / greater than -HRES_ADJ, followed by:
3819         /
3820         /       Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3821         /
3822         / which generates a carry only when step 1 is true and the most
3823         / significant long of the longlong hrestime_adj is -1.
3824 
3825 .CL6:                                   / hrestime_adj is negative
3826         movl    %esi, %ecx
3827         addl    max_hres_adj, %ecx
3828         movl    %edx, %eax
3829         adcl    $0, %eax
3830         jc      .CL7
3831         xor     %ecx, %ecx
3832         subl    max_hres_adj, %ecx      / adj = -(HRES_ADJ);
3833         movl    $-1, %edx
3834         jmp     .CL5
3835 .CL7:
3836         movl    %esi, %ecx              / adj = hrestime_adj;
3837 .CL5:
3838         movl    timedelta, %esi
3839         subl    %ecx, %esi
3840         movl    timedelta+4, %eax
3841         sbbl    %edx, %eax
3842         movl    %esi, timedelta
3843         movl    %eax, timedelta+4       / timedelta -= adj;
3844         movl    %esi, hrestime_adj
3845         movl    %eax, hrestime_adj+4    / hrestime_adj = timedelta;
3846         addl    hrestime+4, %ecx
3847 
3848         movl    %ecx, %eax              / eax = tv_nsec
3849 1:
3850         cmpl    $NANOSEC, %eax          / if ((unsigned long)tv_nsec >= NANOSEC)
3851         jb      .CL8                    / no
3852         incl    one_sec                 / yes,  one_sec++;
3853         incl    hrestime                / hrestime.tv_sec++;
3854         addl    $-NANOSEC, %eax         / tv_nsec -= NANOSEC
3855         jmp     1b                      / check for more seconds
3856 
3857 .CL8:
3858         movl    %eax, hrestime+4        / store final into hrestime.tv_nsec
3859         incl    hres_lock               / release the hres_lock
3860 
3861         popl    %ebx
3862         popl    %esi
3863         leave
3864         ret
3865         SET_SIZE(hres_tick)
3866 
3867 #endif  /* __i386 */
3868 #endif  /* __lint */
3869 
3870 /*
3871  * void prefetch_smap_w(void *)
3872  *
3873  * Prefetch ahead within a linear list of smap structures.
3874  * Not implemented for ia32.  Stub for compatibility.
3875  */
3876 
3877 #if defined(__lint)
3878 
3879 /*ARGSUSED*/
3880 void prefetch_smap_w(void *smp)
3881 {}
3882 
3883 #else   /* __lint */
3884 
3885         ENTRY(prefetch_smap_w)
3886         rep;    ret     /* use 2 byte return instruction when branch target */
3887                         /* AMD Software Optimization Guide - Section 6.2 */
3888         SET_SIZE(prefetch_smap_w)
3889 
3890 #endif  /* __lint */
3891 
3892 /*
3893  * prefetch_page_r(page_t *)
3894  * issue prefetch instructions for a page_t
3895  */
3896 #if defined(__lint)
3897 
3898 /*ARGSUSED*/
3899 void
3900 prefetch_page_r(void *pp)
3901 {}
3902 
3903 #else   /* __lint */
3904 
3905         ENTRY(prefetch_page_r)
3906         rep;    ret     /* use 2 byte return instruction when branch target */
3907                         /* AMD Software Optimization Guide - Section 6.2 */
3908         SET_SIZE(prefetch_page_r)
3909 
3910 #endif  /* __lint */
3911 
3912 #if defined(__lint)
3913 
3914 /*ARGSUSED*/
3915 int
3916 bcmp(const void *s1, const void *s2, size_t count)
3917 { return (0); }
3918 
3919 #else   /* __lint */
3920 
3921 #if defined(__amd64)
3922 
3923         ENTRY(bcmp)
3924         pushq   %rbp
3925         movq    %rsp, %rbp
3926 #ifdef DEBUG
3927         testq   %rdx,%rdx
3928         je      1f
3929         movq    postbootkernelbase(%rip), %r11
3930         cmpq    %r11, %rdi
3931         jb      0f
3932         cmpq    %r11, %rsi
3933         jnb     1f
3934 0:      leaq    .bcmp_panic_msg(%rip), %rdi
3935         xorl    %eax, %eax
3936         call    panic
3937 1:
3938 #endif  /* DEBUG */
3939         call    memcmp
3940         testl   %eax, %eax
3941         setne   %dl
3942         leave
3943         movzbl  %dl, %eax
3944         ret
3945         SET_SIZE(bcmp)
3946         
3947 #elif defined(__i386)
3948         
3949 #define ARG_S1          8
3950 #define ARG_S2          12
3951 #define ARG_LENGTH      16
3952 
3953         ENTRY(bcmp)
3954         pushl   %ebp
3955         movl    %esp, %ebp      / create new stack frame
3956 #ifdef DEBUG
3957         cmpl    $0, ARG_LENGTH(%ebp)
3958         je      1f
3959         movl    postbootkernelbase, %eax
3960         cmpl    %eax, ARG_S1(%ebp)
3961         jb      0f
3962         cmpl    %eax, ARG_S2(%ebp)
3963         jnb     1f
3964 0:      pushl   $.bcmp_panic_msg
3965         call    panic
3966 1:
3967 #endif  /* DEBUG */
3968 
3969         pushl   %edi            / save register variable
3970         movl    ARG_S1(%ebp), %eax      / %eax = address of string 1
3971         movl    ARG_S2(%ebp), %ecx      / %ecx = address of string 2
3972         cmpl    %eax, %ecx      / if the same string
3973         je      .equal          / goto .equal
3974         movl    ARG_LENGTH(%ebp), %edi  / %edi = length in bytes
3975         cmpl    $4, %edi        / if %edi < 4
3976         jb      .byte_check     / goto .byte_check
3977         .align  4
3978 .word_loop:
3979         movl    (%ecx), %edx    / move 1 word from (%ecx) to %edx
3980         leal    -4(%edi), %edi  / %edi -= 4
3981         cmpl    (%eax), %edx    / compare 1 word from (%eax) with %edx
3982         jne     .word_not_equal / if not equal, goto .word_not_equal
3983         leal    4(%ecx), %ecx   / %ecx += 4 (next word)
3984         leal    4(%eax), %eax   / %eax += 4 (next word)
3985         cmpl    $4, %edi        / if %edi >= 4
3986         jae     .word_loop      / goto .word_loop
3987 .byte_check:
3988         cmpl    $0, %edi        / if %edi == 0
3989         je      .equal          / goto .equal
3990         jmp     .byte_loop      / goto .byte_loop (checks in bytes)
3991 .word_not_equal:
3992         leal    4(%edi), %edi   / %edi += 4 (post-decremented)
3993         .align  4
3994 .byte_loop:
3995         movb    (%ecx), %dl     / move 1 byte from (%ecx) to %dl
3996         cmpb    %dl, (%eax)     / compare %dl with 1 byte from (%eax)
3997         jne     .not_equal      / if not equal, goto .not_equal
3998         incl    %ecx            / %ecx++ (next byte)
3999         incl    %eax            / %eax++ (next byte)
4000         decl    %edi            / %edi--
4001         jnz     .byte_loop      / if not zero, goto .byte_loop
4002 .equal:
4003         xorl    %eax, %eax      / %eax = 0
4004         popl    %edi            / restore register variable
4005         leave                   / restore old stack frame
4006         ret                     / return (NULL)
4007         .align  4
4008 .not_equal:
4009         movl    $1, %eax        / return 1
4010         popl    %edi            / restore register variable
4011         leave                   / restore old stack frame
4012         ret                     / return (NULL)
4013         SET_SIZE(bcmp)
4014 
4015 #endif  /* __i386 */
4016 
4017 #ifdef DEBUG
4018         .text
4019 .bcmp_panic_msg:
4020         .string "bcmp: arguments below kernelbase"
4021 #endif  /* DEBUG */
4022 
4023 #endif  /* __lint */
4024 
4025 #if defined(__lint)
4026 
4027 uint_t
4028 bsrw_insn(uint16_t mask)
4029 {
4030         uint_t index = sizeof (mask) * NBBY - 1;
4031 
4032         while ((mask & (1 << index)) == 0)
4033                 index--;
4034         return (index);
4035 }
4036 
4037 #else   /* __lint */
4038 
4039 #if defined(__amd64)
4040 
4041         ENTRY_NP(bsrw_insn)
4042         xorl    %eax, %eax
4043         bsrw    %di, %ax
4044         ret
4045         SET_SIZE(bsrw_insn)
4046 
4047 #elif defined(__i386)
4048 
4049         ENTRY_NP(bsrw_insn)
4050         movw    4(%esp), %cx
4051         xorl    %eax, %eax
4052         bsrw    %cx, %ax
4053         ret
4054         SET_SIZE(bsrw_insn)
4055 
4056 #endif  /* __i386 */
4057 #endif  /* __lint */
4058 
4059 #if defined(__lint)
4060 
4061 uint_t
4062 atomic_btr32(uint32_t *pending, uint_t pil)
4063 {
4064         return (*pending &= ~(1 << pil));
4065 }
4066 
4067 #else   /* __lint */
4068 
4069 #if defined(__i386)
4070 
4071         ENTRY_NP(atomic_btr32)
4072         movl    4(%esp), %ecx
4073         movl    8(%esp), %edx
4074         xorl    %eax, %eax
4075         lock
4076         btrl    %edx, (%ecx)
4077         setc    %al
4078         ret
4079         SET_SIZE(atomic_btr32)
4080 
4081 #endif  /* __i386 */
4082 #endif  /* __lint */
4083 
4084 #if defined(__lint)
4085 
4086 /*ARGSUSED*/
4087 void
4088 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4089             uint_t arg2)
4090 {}
4091 
4092 #else   /* __lint */
4093 
4094 #if defined(__amd64)
4095 
4096         ENTRY_NP(switch_sp_and_call)
4097         pushq   %rbp
4098         movq    %rsp, %rbp              /* set up stack frame */
4099         movq    %rdi, %rsp              /* switch stack pointer */
4100         movq    %rdx, %rdi              /* pass func arg 1 */
4101         movq    %rsi, %r11              /* save function to call */
4102         movq    %rcx, %rsi              /* pass func arg 2 */
4103         call    *%r11                   /* call function */
4104         leave                           /* restore stack */
4105         ret
4106         SET_SIZE(switch_sp_and_call)
4107 
4108 #elif defined(__i386)
4109 
4110         ENTRY_NP(switch_sp_and_call)
4111         pushl   %ebp
4112         mov     %esp, %ebp              /* set up stack frame */
4113         movl    8(%ebp), %esp           /* switch stack pointer */
4114         pushl   20(%ebp)                /* push func arg 2 */
4115         pushl   16(%ebp)                /* push func arg 1 */
4116         call    *12(%ebp)               /* call function */
4117         addl    $8, %esp                /* pop arguments */
4118         leave                           /* restore stack */
4119         ret
4120         SET_SIZE(switch_sp_and_call)
4121 
4122 #endif  /* __i386 */
4123 #endif  /* __lint */
4124 
4125 #if defined(__lint)
4126 
4127 void
4128 kmdb_enter(void)
4129 {}
4130 
4131 #else   /* __lint */
4132 
4133 #if defined(__amd64)
4134 
4135         ENTRY_NP(kmdb_enter)
4136         pushq   %rbp
4137         movq    %rsp, %rbp
4138 
4139         /*
4140          * Save flags, do a 'cli' then return the saved flags
4141          */
4142         call    intr_clear
4143 
4144         int     $T_DBGENTR
4145 
4146         /*
4147          * Restore the saved flags
4148          */
4149         movq    %rax, %rdi
4150         call    intr_restore
4151 
4152         leave
4153         ret     
4154         SET_SIZE(kmdb_enter)
4155 
4156 #elif defined(__i386)
4157 
4158         ENTRY_NP(kmdb_enter)
4159         pushl   %ebp
4160         movl    %esp, %ebp
4161 
4162         /*
4163          * Save flags, do a 'cli' then return the saved flags
4164          */
4165         call    intr_clear
4166 
4167         int     $T_DBGENTR
4168 
4169         /*
4170          * Restore the saved flags
4171          */
4172         pushl   %eax
4173         call    intr_restore
4174         addl    $4, %esp
4175 
4176         leave
4177         ret     
4178         SET_SIZE(kmdb_enter)
4179 
4180 #endif  /* __i386 */
4181 #endif  /* __lint */
4182 
4183 #if defined(__lint)
4184 
4185 void
4186 return_instr(void)
4187 {}
4188 
4189 #else   /* __lint */
4190 
4191         ENTRY_NP(return_instr)
4192         rep;    ret     /* use 2 byte instruction when branch target */
4193                         /* AMD Software Optimization Guide - Section 6.2 */
4194         SET_SIZE(return_instr)
4195 
4196 #endif  /* __lint */
4197 
4198 #if defined(__lint)
4199 
4200 ulong_t
4201 getflags(void)
4202 {
4203         return (0);
4204 }
4205 
4206 #else   /* __lint */
4207 
4208 #if defined(__amd64)
4209 
4210         ENTRY(getflags)
4211         pushfq
4212         popq    %rax
4213 #if defined(__xpv)
4214         CURTHREAD(%rdi)
4215         KPREEMPT_DISABLE(%rdi)
4216         /*
4217          * Synthesize the PS_IE bit from the event mask bit
4218          */
4219         CURVCPU(%r11)
4220         andq    $_BITNOT(PS_IE), %rax
4221         XEN_TEST_UPCALL_MASK(%r11)
4222         jnz     1f
4223         orq     $PS_IE, %rax
4224 1:
4225         KPREEMPT_ENABLE_NOKP(%rdi)
4226 #endif
4227         ret
4228         SET_SIZE(getflags)
4229 
4230 #elif defined(__i386)
4231 
4232         ENTRY(getflags)
4233         pushfl
4234         popl    %eax
4235 #if defined(__xpv)
4236         CURTHREAD(%ecx)
4237         KPREEMPT_DISABLE(%ecx)
4238         /*
4239          * Synthesize the PS_IE bit from the event mask bit
4240          */
4241         CURVCPU(%edx)
4242         andl    $_BITNOT(PS_IE), %eax
4243         XEN_TEST_UPCALL_MASK(%edx)
4244         jnz     1f
4245         orl     $PS_IE, %eax
4246 1:
4247         KPREEMPT_ENABLE_NOKP(%ecx)
4248 #endif
4249         ret
4250         SET_SIZE(getflags)
4251 
4252 #endif  /* __i386 */
4253 
4254 #endif  /* __lint */
4255 
4256 #if defined(__lint)
4257 
4258 ftrace_icookie_t
4259 ftrace_interrupt_disable(void)
4260 { return (0); }
4261 
4262 #else   /* __lint */
4263 
4264 #if defined(__amd64)
4265 
4266         ENTRY(ftrace_interrupt_disable)
4267         pushfq
4268         popq    %rax
4269         CLI(%rdx)
4270         ret
4271         SET_SIZE(ftrace_interrupt_disable)
4272 
4273 #elif defined(__i386)
4274                 
4275         ENTRY(ftrace_interrupt_disable)
4276         pushfl
4277         popl    %eax
4278         CLI(%edx)
4279         ret
4280         SET_SIZE(ftrace_interrupt_disable)
4281 
4282 #endif  /* __i386 */    
4283 #endif  /* __lint */
4284 
4285 #if defined(__lint)
4286 
4287 /*ARGSUSED*/
4288 void
4289 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4290 {}
4291 
4292 #else   /* __lint */
4293 
4294 #if defined(__amd64)
4295 
4296         ENTRY(ftrace_interrupt_enable)
4297         pushq   %rdi
4298         popfq
4299         ret
4300         SET_SIZE(ftrace_interrupt_enable)
4301 
4302 #elif defined(__i386)
4303                 
4304         ENTRY(ftrace_interrupt_enable)
4305         movl    4(%esp), %eax
4306         pushl   %eax
4307         popfl
4308         ret
4309         SET_SIZE(ftrace_interrupt_enable)
4310 
4311 #endif  /* __i386 */
4312 #endif  /* __lint */
4313 
4314 #if defined (__lint)
4315 
4316 /*ARGSUSED*/
4317 void
4318 clflush_insn(caddr_t addr)
4319 {}
4320 
4321 #else /* __lint */
4322 
4323 #if defined (__amd64)
4324         ENTRY(clflush_insn)
4325         clflush (%rdi)
4326         ret
4327         SET_SIZE(clflush_insn)
4328 #elif defined (__i386)
4329         ENTRY(clflush_insn)
4330         movl    4(%esp), %eax
4331         clflush (%eax)
4332         ret
4333         SET_SIZE(clflush_insn)
4334 
4335 #endif /* __i386 */
4336 #endif /* __lint */
4337 
4338 #if defined (__lint)
4339 /*ARGSUSED*/
4340 void
4341 mfence_insn(void)
4342 {}
4343 
4344 #else /* __lint */
4345 
4346 #if defined (__amd64)
4347         ENTRY(mfence_insn)
4348         mfence
4349         ret
4350         SET_SIZE(mfence_insn)
4351 #elif defined (__i386)
4352         ENTRY(mfence_insn)
4353         mfence
4354         ret
4355         SET_SIZE(mfence_insn)
4356 
4357 #endif /* __i386 */
4358 #endif /* __lint */
4359 
4360 /*
4361  * VMware implements an I/O port that programs can query to detect if software
4362  * is running in a VMware hypervisor. This hypervisor port behaves differently
4363  * depending on magic values in certain registers and modifies some registers
4364  * as a side effect.
4365  *
4366  * References: http://kb.vmware.com/kb/1009458 
4367  */
4368 
4369 #if defined(__lint)
4370 
4371 /* ARGSUSED */
4372 void
4373 vmware_port(int cmd, uint32_t *regs) { return; }
4374 
4375 #else
4376 
4377 #if defined(__amd64)
4378 
4379         ENTRY(vmware_port)
4380         pushq   %rbx
4381         movl    $VMWARE_HVMAGIC, %eax
4382         movl    $0xffffffff, %ebx
4383         movl    %edi, %ecx
4384         movl    $VMWARE_HVPORT, %edx
4385         inl     (%dx)
4386         movl    %eax, (%rsi)
4387         movl    %ebx, 4(%rsi)
4388         movl    %ecx, 8(%rsi)
4389         movl    %edx, 12(%rsi)
4390         popq    %rbx
4391         ret
4392         SET_SIZE(vmware_port)
4393 
4394 #elif defined(__i386)
4395 
4396         ENTRY(vmware_port)
4397         pushl   %ebx
4398         pushl   %esi
4399         movl    $VMWARE_HVMAGIC, %eax
4400         movl    $0xffffffff, %ebx
4401         movl    12(%esp), %ecx
4402         movl    $VMWARE_HVPORT, %edx
4403         inl     (%dx)
4404         movl    16(%esp), %esi
4405         movl    %eax, (%esi)
4406         movl    %ebx, 4(%esi)
4407         movl    %ecx, 8(%esi)
4408         movl    %edx, 12(%esi)
4409         popl    %esi
4410         popl    %ebx
4411         ret
4412         SET_SIZE(vmware_port)
4413 
4414 #endif /* __i386 */
4415 #endif /* __lint */