1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  25  * Copyright (c) 2014 by Delphix. All rights reserved.
  26  * Copyright 2019 Joyent, Inc.
  27  */
  28 
  29 /*
  30  *  Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
  31  *  Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
  32  *    All Rights Reserved
  33  */
  34 
  35 /*
  36  * Copyright (c) 2009, Intel Corporation.
  37  * All rights reserved.
  38  */
  39 
  40 /*
  41  * General assembly language routines.
  42  * It is the intent of this file to contain routines that are
  43  * independent of the specific kernel architecture, and those that are
  44  * common across kernel architectures.
  45  * As architectures diverge, and implementations of specific
  46  * architecture-dependent routines change, the routines should be moved
  47  * from this file into the respective ../`arch -k`/subr.s file.
  48  */
  49 
  50 #include <sys/asm_linkage.h>
  51 #include <sys/asm_misc.h>
  52 #include <sys/panic.h>
  53 #include <sys/ontrap.h>
  54 #include <sys/regset.h>
  55 #include <sys/privregs.h>
  56 #include <sys/reboot.h>
  57 #include <sys/psw.h>
  58 #include <sys/x86_archext.h>
  59 
  60 #include "assym.h"
  61 #include <sys/dditypes.h>
  62 
  63 /*
  64  * on_fault()
  65  *
  66  * Catch lofault faults. Like setjmp except it returns one
  67  * if code following causes uncorrectable fault. Turned off
  68  * by calling no_fault(). Note that while under on_fault(),
  69  * SMAP is disabled. For more information see
  70  * uts/intel/ia32/ml/copy.s.
  71  */
  72 
  73         ENTRY(on_fault)
  74         movq    %gs:CPU_THREAD, %rsi
  75         leaq    catch_fault(%rip), %rdx
  76         movq    %rdi, T_ONFAULT(%rsi)           /* jumpbuf in t_onfault */
  77         movq    %rdx, T_LOFAULT(%rsi)           /* catch_fault in t_lofault */
  78         call    smap_disable                    /* allow user accesses */
  79         jmp     setjmp                          /* let setjmp do the rest */
  80 
  81 catch_fault:
  82         movq    %gs:CPU_THREAD, %rsi
  83         movq    T_ONFAULT(%rsi), %rdi           /* address of save area */
  84         xorl    %eax, %eax
  85         movq    %rax, T_ONFAULT(%rsi)           /* turn off onfault */
  86         movq    %rax, T_LOFAULT(%rsi)           /* turn off lofault */
  87         call    smap_enable                     /* disallow user accesses */
  88         jmp     longjmp                         /* let longjmp do the rest */
  89         SET_SIZE(on_fault)
  90 
  91         ENTRY(no_fault)
  92         movq    %gs:CPU_THREAD, %rsi
  93         xorl    %eax, %eax
  94         movq    %rax, T_ONFAULT(%rsi)           /* turn off onfault */
  95         movq    %rax, T_LOFAULT(%rsi)           /* turn off lofault */
  96         call    smap_enable                     /* disallow user accesses */
  97         ret
  98         SET_SIZE(no_fault)
  99 
 100 /*
 101  * Default trampoline code for on_trap() (see <sys/ontrap.h>).  We just
 102  * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
 103  */
 104 
 105         ENTRY(on_trap_trampoline)
 106         movq    %gs:CPU_THREAD, %rsi
 107         movq    T_ONTRAP(%rsi), %rdi
 108         addq    $OT_JMPBUF, %rdi
 109         jmp     longjmp
 110         SET_SIZE(on_trap_trampoline)
 111 
 112 /*
 113  * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
 114  * more information about the on_trap() mechanism.  If the on_trap_data is the
 115  * same as the topmost stack element, we just modify that element.
 116  */
 117 
 118         ENTRY(on_trap)
 119         movw    %si, OT_PROT(%rdi)              /* ot_prot = prot */
 120         movw    $0, OT_TRAP(%rdi)               /* ot_trap = 0 */
 121         leaq    on_trap_trampoline(%rip), %rdx  /* rdx = &on_trap_trampoline */
 122         movq    %rdx, OT_TRAMPOLINE(%rdi)       /* ot_trampoline = rdx */
 123         xorl    %ecx, %ecx
 124         movq    %rcx, OT_HANDLE(%rdi)           /* ot_handle = NULL */
 125         movq    %rcx, OT_PAD1(%rdi)             /* ot_pad1 = NULL */
 126         movq    %gs:CPU_THREAD, %rdx            /* rdx = curthread */
 127         movq    T_ONTRAP(%rdx), %rcx            /* rcx = curthread->t_ontrap */
 128         cmpq    %rdi, %rcx                      /* if (otp == %rcx)     */
 129         je      0f                              /*      don't modify t_ontrap */
 130 
 131         movq    %rcx, OT_PREV(%rdi)             /* ot_prev = t_ontrap */
 132         movq    %rdi, T_ONTRAP(%rdx)            /* curthread->t_ontrap = otp */
 133 
 134 0:      addq    $OT_JMPBUF, %rdi                /* &ot_jmpbuf */
 135         jmp     setjmp
 136         SET_SIZE(on_trap)
 137 
 138 /*
 139  * Setjmp and longjmp implement non-local gotos using state vectors
 140  * type label_t.
 141  */
 142 
 143 #if LABEL_PC != 0
 144 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
 145 #endif  /* LABEL_PC != 0 */
 146 
 147         ENTRY(setjmp)
 148         movq    %rsp, LABEL_SP(%rdi)
 149         movq    %rbp, LABEL_RBP(%rdi)
 150         movq    %rbx, LABEL_RBX(%rdi)
 151         movq    %r12, LABEL_R12(%rdi)
 152         movq    %r13, LABEL_R13(%rdi)
 153         movq    %r14, LABEL_R14(%rdi)
 154         movq    %r15, LABEL_R15(%rdi)
 155         movq    (%rsp), %rdx            /* return address */
 156         movq    %rdx, (%rdi)            /* LABEL_PC is 0 */
 157         xorl    %eax, %eax              /* return 0 */
 158         ret
 159         SET_SIZE(setjmp)
 160 
 161         ENTRY(longjmp)
 162         movq    LABEL_SP(%rdi), %rsp
 163         movq    LABEL_RBP(%rdi), %rbp
 164         movq    LABEL_RBX(%rdi), %rbx
 165         movq    LABEL_R12(%rdi), %r12
 166         movq    LABEL_R13(%rdi), %r13
 167         movq    LABEL_R14(%rdi), %r14
 168         movq    LABEL_R15(%rdi), %r15
 169         movq    (%rdi), %rdx            /* return address; LABEL_PC is 0 */
 170         movq    %rdx, (%rsp)
 171         xorl    %eax, %eax
 172         incl    %eax                    /* return 1 */
 173         ret
 174         SET_SIZE(longjmp)
 175 
 176 /*
 177  * if a() calls b() calls caller(),
 178  * caller() returns return address in a().
 179  * (Note: We assume a() and b() are C routines which do the normal entry/exit
 180  *  sequence.)
 181  */
 182 
 183         ENTRY(caller)
 184         movq    8(%rbp), %rax           /* b()'s return pc, in a() */
 185         ret
 186         SET_SIZE(caller)
 187 
 188 /*
 189  * if a() calls callee(), callee() returns the
 190  * return address in a();
 191  */
 192 
 193         ENTRY(callee)
 194         movq    (%rsp), %rax            /* callee()'s return pc, in a() */
 195         ret
 196         SET_SIZE(callee)
 197 
 198 /*
 199  * return the current frame pointer
 200  */
 201 
 202         ENTRY(getfp)
 203         movq    %rbp, %rax
 204         ret
 205         SET_SIZE(getfp)
 206 
 207 /*
 208  * Invalidate a single page table entry in the TLB
 209  */
 210 
 211         ENTRY(mmu_invlpg)
 212         invlpg  (%rdi)
 213         ret
 214         SET_SIZE(mmu_invlpg)
 215 
 216 
 217 /*
 218  * Get/Set the value of various control registers
 219  */
 220 
 221         ENTRY(getcr0)
 222         movq    %cr0, %rax
 223         ret
 224         SET_SIZE(getcr0)
 225 
 226         ENTRY(setcr0)
 227         movq    %rdi, %cr0
 228         ret
 229         SET_SIZE(setcr0)
 230 
 231         ENTRY(getcr2)
 232 #if defined(__xpv)
 233         movq    %gs:CPU_VCPU_INFO, %rax
 234         movq    VCPU_INFO_ARCH_CR2(%rax), %rax
 235 #else
 236         movq    %cr2, %rax
 237 #endif
 238         ret
 239         SET_SIZE(getcr2)
 240 
 241         ENTRY(getcr3)
 242         movq    %cr3, %rax
 243         ret
 244         SET_SIZE(getcr3)
 245 
 246 #if !defined(__xpv)
 247 
 248         ENTRY(setcr3)
 249         movq    %rdi, %cr3
 250         ret
 251         SET_SIZE(setcr3)
 252 
 253         ENTRY(reload_cr3)
 254         movq    %cr3, %rdi
 255         movq    %rdi, %cr3
 256         ret
 257         SET_SIZE(reload_cr3)
 258 
 259 #endif  /* __xpv */
 260 
 261         ENTRY(getcr4)
 262         movq    %cr4, %rax
 263         ret
 264         SET_SIZE(getcr4)
 265 
 266         ENTRY(setcr4)
 267         movq    %rdi, %cr4
 268         ret
 269         SET_SIZE(setcr4)
 270 
 271         ENTRY(getcr8)
 272         movq    %cr8, %rax
 273         ret
 274         SET_SIZE(getcr8)
 275 
 276         ENTRY(setcr8)
 277         movq    %rdi, %cr8
 278         ret
 279         SET_SIZE(setcr8)
 280 
 281         ENTRY(__cpuid_insn)
 282         movq    %rbx, %r8
 283         movq    %rcx, %r9
 284         movq    %rdx, %r11
 285         movl    (%rdi), %eax            /* %eax = regs->cp_eax */
 286         movl    0x4(%rdi), %ebx         /* %ebx = regs->cp_ebx */
 287         movl    0x8(%rdi), %ecx         /* %ecx = regs->cp_ecx */
 288         movl    0xc(%rdi), %edx         /* %edx = regs->cp_edx */
 289         cpuid
 290         movl    %eax, (%rdi)            /* regs->cp_eax = %eax */
 291         movl    %ebx, 0x4(%rdi)         /* regs->cp_ebx = %ebx */
 292         movl    %ecx, 0x8(%rdi)         /* regs->cp_ecx = %ecx */
 293         movl    %edx, 0xc(%rdi)         /* regs->cp_edx = %edx */
 294         movq    %r8, %rbx
 295         movq    %r9, %rcx
 296         movq    %r11, %rdx
 297         ret
 298         SET_SIZE(__cpuid_insn)
 299 
 300         ENTRY_NP(i86_monitor)
 301         pushq   %rbp
 302         movq    %rsp, %rbp
 303         movq    %rdi, %rax              /* addr */
 304         movq    %rsi, %rcx              /* extensions */
 305         /* rdx contains input arg3: hints */
 306         clflush (%rax)
 307         .byte   0x0f, 0x01, 0xc8        /* monitor */
 308         leave
 309         ret
 310         SET_SIZE(i86_monitor)
 311 
 312         ENTRY_NP(i86_mwait)
 313         pushq   %rbp
 314         call    x86_md_clear
 315         movq    %rsp, %rbp
 316         movq    %rdi, %rax              /* data */
 317         movq    %rsi, %rcx              /* extensions */
 318         .byte   0x0f, 0x01, 0xc9        /* mwait */
 319         leave
 320         ret
 321         SET_SIZE(i86_mwait)
 322 
 323 #if defined(__xpv)
 324         /*
 325          * Defined in C
 326          */
 327 #else
 328 
 329         ENTRY_NP(tsc_read)
 330         movq    %rbx, %r11
 331         movl    $0, %eax
 332         cpuid
 333         rdtsc
 334         movq    %r11, %rbx
 335         shlq    $32, %rdx
 336         orq     %rdx, %rax
 337         ret
 338         .globl _tsc_mfence_start
 339 _tsc_mfence_start:
 340         mfence
 341         rdtsc
 342         shlq    $32, %rdx
 343         orq     %rdx, %rax
 344         ret
 345         .globl _tsc_mfence_end
 346 _tsc_mfence_end:
 347         .globl _tscp_start
 348 _tscp_start:
 349         .byte   0x0f, 0x01, 0xf9        /* rdtscp instruction */
 350         shlq    $32, %rdx
 351         orq     %rdx, %rax
 352         ret
 353         .globl _tscp_end
 354 _tscp_end:
 355         .globl _no_rdtsc_start
 356 _no_rdtsc_start:
 357         xorl    %edx, %edx
 358         xorl    %eax, %eax
 359         ret
 360         .globl _no_rdtsc_end
 361 _no_rdtsc_end:
 362         .globl _tsc_lfence_start
 363 _tsc_lfence_start:
 364         lfence
 365         rdtsc
 366         shlq    $32, %rdx
 367         orq     %rdx, %rax
 368         ret
 369         .globl _tsc_lfence_end
 370 _tsc_lfence_end:
 371         SET_SIZE(tsc_read)
 372 
 373 
 374 #endif  /* __xpv */
 375 
 376         ENTRY_NP(randtick)
 377         rdtsc
 378         shlq    $32, %rdx
 379         orq     %rdx, %rax
 380         ret
 381         SET_SIZE(randtick)
 382 /*
 383  * Insert entryp after predp in a doubly linked list.
 384  */
 385 
 386         ENTRY(_insque)
 387         movq    (%rsi), %rax            /* predp->forw                       */
 388         movq    %rsi, CPTRSIZE(%rdi)    /* entryp->back = predp              */
 389         movq    %rax, (%rdi)            /* entryp->forw = predp->forw     */
 390         movq    %rdi, (%rsi)            /* predp->forw = entryp              */
 391         movq    %rdi, CPTRSIZE(%rax)    /* predp->forw->back = entryp     */
 392         ret
 393         SET_SIZE(_insque)
 394 
 395 /*
 396  * Remove entryp from a doubly linked list
 397  */
 398 
 399         ENTRY(_remque)
 400         movq    (%rdi), %rax            /* entry->forw */
 401         movq    CPTRSIZE(%rdi), %rdx    /* entry->back */
 402         movq    %rax, (%rdx)            /* entry->back->forw = entry->forw */
 403         movq    %rdx, CPTRSIZE(%rax)    /* entry->forw->back = entry->back */
 404         ret
 405         SET_SIZE(_remque)
 406 
 407 /*
 408  * Returns the number of
 409  * non-NULL bytes in string argument.
 410  */
 411 
 412 /*
 413  * This is close to a simple transliteration of a C version of this
 414  * routine.  We should either just -make- this be a C version, or
 415  * justify having it in assembler by making it significantly faster.
 416  *
 417  * size_t
 418  * strlen(const char *s)
 419  * {
 420  *      const char *s0;
 421  * #if defined(DEBUG)
 422  *      if ((uintptr_t)s < KERNELBASE)
 423  *              panic(.str_panic_msg);
 424  * #endif
 425  *      for (s0 = s; *s; s++)
 426  *              ;
 427  *      return (s - s0);
 428  * }
 429  */
 430 
 431         ENTRY(strlen)
 432 #ifdef DEBUG
 433         movq    postbootkernelbase(%rip), %rax
 434         cmpq    %rax, %rdi
 435         jae     str_valid
 436         pushq   %rbp
 437         movq    %rsp, %rbp
 438         leaq    .str_panic_msg(%rip), %rdi
 439         xorl    %eax, %eax
 440         call    panic
 441 #endif  /* DEBUG */
 442 str_valid:
 443         cmpb    $0, (%rdi)
 444         movq    %rdi, %rax
 445         je      .null_found
 446         .align  4
 447 .strlen_loop:
 448         incq    %rdi
 449         cmpb    $0, (%rdi)
 450         jne     .strlen_loop
 451 .null_found:
 452         subq    %rax, %rdi
 453         movq    %rdi, %rax
 454         ret
 455         SET_SIZE(strlen)
 456 
 457 #ifdef DEBUG
 458         .text
 459 .str_panic_msg:
 460         .string "strlen: argument below kernelbase"
 461 #endif /* DEBUG */
 462 
 463         /*
 464          * Berkeley 4.3 introduced symbolically named interrupt levels
 465          * as a way deal with priority in a machine independent fashion.
 466          * Numbered priorities are machine specific, and should be
 467          * discouraged where possible.
 468          *
 469          * Note, for the machine specific priorities there are
 470          * examples listed for devices that use a particular priority.
 471          * It should not be construed that all devices of that
 472          * type should be at that priority.  It is currently were
 473          * the current devices fit into the priority scheme based
 474          * upon time criticalness.
 475          *
 476          * The underlying assumption of these assignments is that
 477          * IPL 10 is the highest level from which a device
 478          * routine can call wakeup.  Devices that interrupt from higher
 479          * levels are restricted in what they can do.  If they need
 480          * kernels services they should schedule a routine at a lower
 481          * level (via software interrupt) to do the required
 482          * processing.
 483          *
 484          * Examples of this higher usage:
 485          *      Level   Usage
 486          *      14      Profiling clock (and PROM uart polling clock)
 487          *      12      Serial ports
 488          *
 489          * The serial ports request lower level processing on level 6.
 490          *
 491          * Also, almost all splN routines (where N is a number or a
 492          * mnemonic) will do a RAISE(), on the assumption that they are
 493          * never used to lower our priority.
 494          * The exceptions are:
 495          *      spl8()          Because you can't be above 15 to begin with!
 496          *      splzs()         Because this is used at boot time to lower our
 497          *                      priority, to allow the PROM to poll the uart.
 498          *      spl0()          Used to lower priority to 0.
 499          */
 500 
 501 #define SETPRI(level) \
 502         movl    $/**/level, %edi;       /* new priority */              \
 503         jmp     do_splx                 /* redirect to do_splx */
 504 
 505 #define RAISE(level) \
 506         movl    $/**/level, %edi;       /* new priority */              \
 507         jmp     splr                    /* redirect to splr */
 508 
 509         /* locks out all interrupts, including memory errors */
 510         ENTRY(spl8)
 511         SETPRI(15)
 512         SET_SIZE(spl8)
 513 
 514         /* just below the level that profiling runs */
 515         ENTRY(spl7)
 516         RAISE(13)
 517         SET_SIZE(spl7)
 518 
 519         /* sun specific - highest priority onboard serial i/o asy ports */
 520         ENTRY(splzs)
 521         SETPRI(12)      /* Can't be a RAISE, as it's used to lower us */
 522         SET_SIZE(splzs)
 523 
 524         ENTRY(splhi)
 525         ALTENTRY(splhigh)
 526         ALTENTRY(spl6)
 527         ALTENTRY(i_ddi_splhigh)
 528 
 529         RAISE(DISP_LEVEL)
 530 
 531         SET_SIZE(i_ddi_splhigh)
 532         SET_SIZE(spl6)
 533         SET_SIZE(splhigh)
 534         SET_SIZE(splhi)
 535 
 536         /* allow all interrupts */
 537         ENTRY(spl0)
 538         SETPRI(0)
 539         SET_SIZE(spl0)
 540 
 541 
 542         /* splx implementation */
 543         ENTRY(splx)
 544         jmp     do_splx         /* redirect to common splx code */
 545         SET_SIZE(splx)
 546 
 547         ENTRY(wait_500ms)
 548         pushq   %rbx
 549         movl    $50000, %ebx
 550 1:
 551         call    tenmicrosec
 552         decl    %ebx
 553         jnz     1b
 554         popq    %rbx
 555         ret
 556         SET_SIZE(wait_500ms)
 557 
 558 #define RESET_METHOD_KBC        1
 559 #define RESET_METHOD_PORT92     2
 560 #define RESET_METHOD_PCI        4
 561 
 562         DGDEF3(pc_reset_methods, 4, 8)
 563         .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
 564 
 565         ENTRY(pc_reset)
 566 
 567         testl   $RESET_METHOD_KBC, pc_reset_methods(%rip)
 568         jz      1f
 569 
 570         /
 571         / Try the classic keyboard controller-triggered reset.
 572         /
 573         movw    $0x64, %dx
 574         movb    $0xfe, %al
 575         outb    (%dx)
 576 
 577         / Wait up to 500 milliseconds here for the keyboard controller
 578         / to pull the reset line.  On some systems where the keyboard
 579         / controller is slow to pull the reset line, the next reset method
 580         / may be executed (which may be bad if those systems hang when the
 581         / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
 582         / and Ferrari 4000 (doesn't like the cf9 reset method))
 583 
 584         call    wait_500ms
 585 
 586 1:
 587         testl   $RESET_METHOD_PORT92, pc_reset_methods(%rip)
 588         jz      3f
 589 
 590         /
 591         / Try port 0x92 fast reset
 592         /
 593         movw    $0x92, %dx
 594         inb     (%dx)
 595         cmpb    $0xff, %al      / If port's not there, we should get back 0xFF
 596         je      1f
 597         testb   $1, %al         / If bit 0
 598         jz      2f              / is clear, jump to perform the reset
 599         andb    $0xfe, %al      / otherwise,
 600         outb    (%dx)           / clear bit 0 first, then
 601 2:
 602         orb     $1, %al         / Set bit 0
 603         outb    (%dx)           / and reset the system
 604 1:
 605 
 606         call    wait_500ms
 607 
 608 3:
 609         testl   $RESET_METHOD_PCI, pc_reset_methods(%rip)
 610         jz      4f
 611 
 612         / Try the PCI (soft) reset vector (should work on all modern systems,
 613         / but has been shown to cause problems on 450NX systems, and some newer
 614         / systems (e.g. ATI IXP400-equipped systems))
 615         / When resetting via this method, 2 writes are required.  The first
 616         / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
 617         / power cycle).
 618         / The reset occurs on the second write, during bit 2 's transition from
 619         / 0->1.
 620         movw    $0xcf9, %dx
 621         movb    $0x2, %al       / Reset mode = hard, no power cycle
 622         outb    (%dx)
 623         movb    $0x6, %al
 624         outb    (%dx)
 625 
 626         call    wait_500ms
 627 
 628 4:
 629         /
 630         / port 0xcf9 failed also.  Last-ditch effort is to
 631         / triple-fault the CPU.
 632         / Also, use triple fault for EFI firmware
 633         /
 634         ENTRY(efi_reset)
 635         pushq   $0x0
 636         pushq   $0x0            / IDT base of 0, limit of 0 + 2 unused bytes
 637         lidt    (%rsp)
 638         int     $0x0            / Trigger interrupt, generate triple-fault
 639 
 640         cli
 641         hlt                     / Wait forever
 642         /*NOTREACHED*/
 643         SET_SIZE(efi_reset)
 644         SET_SIZE(pc_reset)
 645 
 646 /*
 647  * C callable in and out routines
 648  */
 649 
 650         ENTRY(outl)
 651         movw    %di, %dx
 652         movl    %esi, %eax
 653         outl    (%dx)
 654         ret
 655         SET_SIZE(outl)
 656 
 657         ENTRY(outw)
 658         movw    %di, %dx
 659         movw    %si, %ax
 660         D16 outl (%dx)          /* XX64 why not outw? */
 661         ret
 662         SET_SIZE(outw)
 663 
 664         ENTRY(outb)
 665         movw    %di, %dx
 666         movb    %sil, %al
 667         outb    (%dx)
 668         ret
 669         SET_SIZE(outb)
 670 
 671         ENTRY(inl)
 672         xorl    %eax, %eax
 673         movw    %di, %dx
 674         inl     (%dx)
 675         ret
 676         SET_SIZE(inl)
 677 
 678         ENTRY(inw)
 679         xorl    %eax, %eax
 680         movw    %di, %dx
 681         D16 inl (%dx)
 682         ret
 683         SET_SIZE(inw)
 684 
 685 
 686         ENTRY(inb)
 687         xorl    %eax, %eax
 688         movw    %di, %dx
 689         inb     (%dx)
 690         ret
 691         SET_SIZE(inb)
 692 
 693 
 694         ENTRY(repoutsw)
 695         movl    %edx, %ecx
 696         movw    %di, %dx
 697         rep
 698           D16 outsl
 699         ret
 700         SET_SIZE(repoutsw)
 701 
 702 
 703         ENTRY(repinsw)
 704         movl    %edx, %ecx
 705         movw    %di, %dx
 706         rep
 707           D16 insl
 708         ret
 709         SET_SIZE(repinsw)
 710 
 711 
 712         ENTRY(repinsb)
 713         movl    %edx, %ecx
 714         movw    %di, %dx
 715         movq    %rsi, %rdi
 716         rep
 717           insb
 718         ret
 719         SET_SIZE(repinsb)
 720 
 721 
 722 /*
 723  * Input a stream of 32-bit words.
 724  * NOTE: count is a DWORD count.
 725  */
 726 
 727         ENTRY(repinsd)
 728         movl    %edx, %ecx
 729         movw    %di, %dx
 730         movq    %rsi, %rdi
 731         rep
 732           insl
 733         ret
 734         SET_SIZE(repinsd)
 735 
 736 /*
 737  * Output a stream of bytes
 738  * NOTE: count is a byte count
 739  */
 740 
 741         ENTRY(repoutsb)
 742         movl    %edx, %ecx
 743         movw    %di, %dx
 744         rep
 745           outsb
 746         ret
 747         SET_SIZE(repoutsb)
 748 
 749 /*
 750  * Output a stream of 32-bit words
 751  * NOTE: count is a DWORD count
 752  */
 753 
 754         ENTRY(repoutsd)
 755         movl    %edx, %ecx
 756         movw    %di, %dx
 757         rep
 758           outsl
 759         ret
 760         SET_SIZE(repoutsd)
 761 
 762 /*
 763  * void int3(void)
 764  * void int18(void)
 765  * void int20(void)
 766  * void int_cmci(void)
 767  */
 768 
 769         ENTRY(int3)
 770         int     $T_BPTFLT
 771         ret
 772         SET_SIZE(int3)
 773 
 774         ENTRY(int18)
 775         int     $T_MCE
 776         ret
 777         SET_SIZE(int18)
 778 
 779         ENTRY(int20)
 780         movl    boothowto, %eax
 781         andl    $RB_DEBUG, %eax
 782         jz      1f
 783 
 784         int     $T_DBGENTR
 785 1:
 786         rep;    ret     /* use 2 byte return instruction when branch target */
 787                         /* AMD Software Optimization Guide - Section 6.2 */
 788         SET_SIZE(int20)
 789 
 790         ENTRY(int_cmci)
 791         int     $T_ENOEXTFLT
 792         ret
 793         SET_SIZE(int_cmci)
 794 
 795         ENTRY(scanc)
 796                                         /* rdi == size */
 797                                         /* rsi == cp */
 798                                         /* rdx == table */
 799                                         /* rcx == mask */
 800         addq    %rsi, %rdi              /* end = &cp[size] */
 801 .scanloop:
 802         cmpq    %rdi, %rsi              /* while (cp < end */
 803         jnb     .scandone
 804         movzbq  (%rsi), %r8             /* %r8 = *cp */
 805         incq    %rsi                    /* cp++ */
 806         testb   %cl, (%r8, %rdx)
 807         jz      .scanloop               /*  && (table[*cp] & mask) == 0) */
 808         decq    %rsi                    /* (fix post-increment) */
 809 .scandone:
 810         movl    %edi, %eax
 811         subl    %esi, %eax              /* return (end - cp) */
 812         ret
 813         SET_SIZE(scanc)
 814 
 815 /*
 816  * Replacement functions for ones that are normally inlined.
 817  * In addition to the copy in i86.il, they are defined here just in case.
 818  */
 819 
 820         ENTRY(intr_clear)
 821         ENTRY(clear_int_flag)
 822         pushfq
 823         popq    %rax
 824 #if defined(__xpv)
 825         leaq    xpv_panicking, %rdi
 826         movl    (%rdi), %edi
 827         cmpl    $0, %edi
 828         jne     2f
 829         CLIRET(%rdi, %dl)       /* returns event mask in %dl */
 830         /*
 831          * Synthesize the PS_IE bit from the event mask bit
 832          */
 833         andq    $_BITNOT(PS_IE), %rax
 834         testb   $1, %dl
 835         jnz     1f
 836         orq     $PS_IE, %rax
 837 1:
 838         ret
 839 2:
 840 #endif
 841         CLI(%rdi)
 842         ret
 843         SET_SIZE(clear_int_flag)
 844         SET_SIZE(intr_clear)
 845 
 846         ENTRY(curcpup)
 847         movq    %gs:CPU_SELF, %rax
 848         ret
 849         SET_SIZE(curcpup)
 850 
 851 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
 852  * These functions reverse the byte order of the input parameter and returns
 853  * the result.  This is to convert the byte order from host byte order
 854  * (little endian) to network byte order (big endian), or vice versa.
 855  */
 856 
 857         ENTRY(htonll)
 858         ALTENTRY(ntohll)
 859         movq    %rdi, %rax
 860         bswapq  %rax
 861         ret
 862         SET_SIZE(ntohll)
 863         SET_SIZE(htonll)
 864 
 865         /* XX64 there must be shorter sequences for this */
 866         ENTRY(htonl)
 867         ALTENTRY(ntohl)
 868         movl    %edi, %eax
 869         bswap   %eax
 870         ret
 871         SET_SIZE(ntohl)
 872         SET_SIZE(htonl)
 873 
 874         /* XX64 there must be better sequences for this */
 875         ENTRY(htons)
 876         ALTENTRY(ntohs)
 877         movl    %edi, %eax
 878         bswap   %eax
 879         shrl    $16, %eax
 880         ret
 881         SET_SIZE(ntohs)
 882         SET_SIZE(htons)
 883 
 884 
 885         ENTRY(intr_restore)
 886         ENTRY(restore_int_flag)
 887         testq   $PS_IE, %rdi
 888         jz      1f
 889 #if defined(__xpv)
 890         leaq    xpv_panicking, %rsi
 891         movl    (%rsi), %esi
 892         cmpl    $0, %esi
 893         jne     1f
 894         /*
 895          * Since we're -really- running unprivileged, our attempt
 896          * to change the state of the IF bit will be ignored.
 897          * The virtual IF bit is tweaked by CLI and STI.
 898          */
 899         IE_TO_EVENT_MASK(%rsi, %rdi)
 900 #else
 901         sti
 902 #endif
 903 1:
 904         ret
 905         SET_SIZE(restore_int_flag)
 906         SET_SIZE(intr_restore)
 907 
 908         ENTRY(sti)
 909         STI
 910         ret
 911         SET_SIZE(sti)
 912 
 913         ENTRY(cli)
 914         CLI(%rax)
 915         ret
 916         SET_SIZE(cli)
 917 
 918         ENTRY(dtrace_interrupt_disable)
 919         pushfq
 920         popq    %rax
 921 #if defined(__xpv)
 922         leaq    xpv_panicking, %rdi
 923         movl    (%rdi), %edi
 924         cmpl    $0, %edi
 925         jne     .dtrace_interrupt_disable_done
 926         CLIRET(%rdi, %dl)       /* returns event mask in %dl */
 927         /*
 928          * Synthesize the PS_IE bit from the event mask bit
 929          */
 930         andq    $_BITNOT(PS_IE), %rax
 931         testb   $1, %dl
 932         jnz     .dtrace_interrupt_disable_done
 933         orq     $PS_IE, %rax
 934 #else
 935         CLI(%rdx)
 936 #endif
 937 .dtrace_interrupt_disable_done:
 938         ret
 939         SET_SIZE(dtrace_interrupt_disable)
 940 
 941         ENTRY(dtrace_interrupt_enable)
 942         pushq   %rdi
 943         popfq
 944 #if defined(__xpv)
 945         leaq    xpv_panicking, %rdx
 946         movl    (%rdx), %edx
 947         cmpl    $0, %edx
 948         jne     .dtrace_interrupt_enable_done
 949         /*
 950          * Since we're -really- running unprivileged, our attempt
 951          * to change the state of the IF bit will be ignored. The
 952          * virtual IF bit is tweaked by CLI and STI.
 953          */
 954         IE_TO_EVENT_MASK(%rdx, %rdi)
 955 #endif
 956 .dtrace_interrupt_enable_done:
 957         ret
 958         SET_SIZE(dtrace_interrupt_enable)
 959 
 960 
 961         ENTRY(dtrace_membar_producer)
 962         rep;    ret     /* use 2 byte return instruction when branch target */
 963                         /* AMD Software Optimization Guide - Section 6.2 */
 964         SET_SIZE(dtrace_membar_producer)
 965 
 966         ENTRY(dtrace_membar_consumer)
 967         rep;    ret     /* use 2 byte return instruction when branch target */
 968                         /* AMD Software Optimization Guide - Section 6.2 */
 969         SET_SIZE(dtrace_membar_consumer)
 970 
 971         ENTRY(threadp)
 972         movq    %gs:CPU_THREAD, %rax
 973         ret
 974         SET_SIZE(threadp)
 975 
 976 /*
 977  *   Checksum routine for Internet Protocol Headers
 978  */
 979 
 980         ENTRY(ip_ocsum)
 981         pushq   %rbp
 982         movq    %rsp, %rbp
 983 #ifdef DEBUG
 984         movq    postbootkernelbase(%rip), %rax
 985         cmpq    %rax, %rdi
 986         jnb     1f
 987         xorl    %eax, %eax
 988         movq    %rdi, %rsi
 989         leaq    .ip_ocsum_panic_msg(%rip), %rdi
 990         call    panic
 991         /*NOTREACHED*/
 992 .ip_ocsum_panic_msg:
 993         .string "ip_ocsum: address 0x%p below kernelbase\n"
 994 1:
 995 #endif
 996         movl    %esi, %ecx      /* halfword_count */
 997         movq    %rdi, %rsi      /* address */
 998                                 /* partial sum in %edx */
 999         xorl    %eax, %eax
1000         testl   %ecx, %ecx
1001         jz      .ip_ocsum_done
1002         testq   $3, %rsi
1003         jnz     .ip_csum_notaligned
1004 .ip_csum_aligned:       /* XX64 opportunities for 8-byte operations? */
1005 .next_iter:
1006         /* XX64 opportunities for prefetch? */
1007         /* XX64 compute csum with 64 bit quantities? */
1008         subl    $32, %ecx
1009         jl      .less_than_32
1010 
1011         addl    0(%rsi), %edx
1012 .only60:
1013         adcl    4(%rsi), %eax
1014 .only56:
1015         adcl    8(%rsi), %edx
1016 .only52:
1017         adcl    12(%rsi), %eax
1018 .only48:
1019         adcl    16(%rsi), %edx
1020 .only44:
1021         adcl    20(%rsi), %eax
1022 .only40:
1023         adcl    24(%rsi), %edx
1024 .only36:
1025         adcl    28(%rsi), %eax
1026 .only32:
1027         adcl    32(%rsi), %edx
1028 .only28:
1029         adcl    36(%rsi), %eax
1030 .only24:
1031         adcl    40(%rsi), %edx
1032 .only20:
1033         adcl    44(%rsi), %eax
1034 .only16:
1035         adcl    48(%rsi), %edx
1036 .only12:
1037         adcl    52(%rsi), %eax
1038 .only8:
1039         adcl    56(%rsi), %edx
1040 .only4:
1041         adcl    60(%rsi), %eax  /* could be adding -1 and -1 with a carry */
1042 .only0:
1043         adcl    $0, %eax        /* could be adding -1 in eax with a carry */
1044         adcl    $0, %eax
1045 
1046         addq    $64, %rsi
1047         testl   %ecx, %ecx
1048         jnz     .next_iter
1049 
1050 .ip_ocsum_done:
1051         addl    %eax, %edx
1052         adcl    $0, %edx
1053         movl    %edx, %eax      /* form a 16 bit checksum by */
1054         shrl    $16, %eax       /* adding two halves of 32 bit checksum */
1055         addw    %dx, %ax
1056         adcw    $0, %ax
1057         andl    $0xffff, %eax
1058         leave
1059         ret
1060 
1061 .ip_csum_notaligned:
1062         xorl    %edi, %edi
1063         movw    (%rsi), %di
1064         addl    %edi, %edx
1065         adcl    $0, %edx
1066         addq    $2, %rsi
1067         decl    %ecx
1068         jmp     .ip_csum_aligned
1069 
1070 .less_than_32:
1071         addl    $32, %ecx
1072         testl   $1, %ecx
1073         jz      .size_aligned
1074         andl    $0xfe, %ecx
1075         movzwl  (%rsi, %rcx, 2), %edi
1076         addl    %edi, %edx
1077         adcl    $0, %edx
1078 .size_aligned:
1079         movl    %ecx, %edi
1080         shrl    $1, %ecx
1081         shl     $1, %edi
1082         subq    $64, %rdi
1083         addq    %rdi, %rsi
1084         leaq    .ip_ocsum_jmptbl(%rip), %rdi
1085         leaq    (%rdi, %rcx, 8), %rdi
1086         xorl    %ecx, %ecx
1087         clc
1088         movq    (%rdi), %rdi
1089         INDIRECT_JMP_REG(rdi)
1090 
1091         .align  8
1092 .ip_ocsum_jmptbl:
1093         .quad   .only0, .only4, .only8, .only12, .only16, .only20
1094         .quad   .only24, .only28, .only32, .only36, .only40, .only44
1095         .quad   .only48, .only52, .only56, .only60
1096         SET_SIZE(ip_ocsum)
1097 
1098 /*
1099  * multiply two long numbers and yield a u_longlong_t result, callable from C.
1100  * Provided to manipulate hrtime_t values.
1101  */
1102 
1103         ENTRY(mul32)
1104         xorl    %edx, %edx      /* XX64 joe, paranoia? */
1105         movl    %edi, %eax
1106         mull    %esi
1107         shlq    $32, %rdx
1108         orq     %rdx, %rax
1109         ret
1110         SET_SIZE(mul32)
1111 
1112         ENTRY(scan_memory)
1113         shrq    $3, %rsi        /* convert %rsi from byte to quadword count */
1114         jz      .scanm_done
1115         movq    %rsi, %rcx      /* move count into rep control register */
1116         movq    %rdi, %rsi      /* move addr into lodsq control reg. */
1117         rep lodsq               /* scan the memory range */
1118 .scanm_done:
1119         rep;    ret     /* use 2 byte return instruction when branch target */
1120                         /* AMD Software Optimization Guide - Section 6.2 */
1121         SET_SIZE(scan_memory)
1122 
1123 
1124         ENTRY(lowbit)
1125         movl    $-1, %eax
1126         bsfq    %rdi, %rdi
1127         cmovnz  %edi, %eax
1128         incl    %eax
1129         ret
1130         SET_SIZE(lowbit)
1131 
1132         ENTRY(highbit)
1133         ALTENTRY(highbit64)
1134         movl    $-1, %eax
1135         bsrq    %rdi, %rdi
1136         cmovnz  %edi, %eax
1137         incl    %eax
1138         ret
1139         SET_SIZE(highbit64)
1140         SET_SIZE(highbit)
1141 
1142 #define XMSR_ACCESS_VAL         $0x9c5a203a
1143 
1144         ENTRY(rdmsr)
1145         movl    %edi, %ecx
1146         rdmsr
1147         shlq    $32, %rdx
1148         orq     %rdx, %rax
1149         ret
1150         SET_SIZE(rdmsr)
1151 
1152         ENTRY(wrmsr)
1153         movq    %rsi, %rdx
1154         shrq    $32, %rdx
1155         movl    %esi, %eax
1156         movl    %edi, %ecx
1157         wrmsr
1158         ret
1159         SET_SIZE(wrmsr)
1160 
1161         ENTRY(xrdmsr)
1162         pushq   %rbp
1163         movq    %rsp, %rbp
1164         movl    %edi, %ecx
1165         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
1166         rdmsr
1167         shlq    $32, %rdx
1168         orq     %rdx, %rax
1169         leave
1170         ret
1171         SET_SIZE(xrdmsr)
1172 
1173         ENTRY(xwrmsr)
1174         pushq   %rbp
1175         movq    %rsp, %rbp
1176         movl    %edi, %ecx
1177         movl    XMSR_ACCESS_VAL, %edi   /* this value is needed to access MSR */
1178         movq    %rsi, %rdx
1179         shrq    $32, %rdx
1180         movl    %esi, %eax
1181         wrmsr
1182         leave
1183         ret
1184         SET_SIZE(xwrmsr)
1185 
1186         ENTRY(get_xcr)
1187         movl    %edi, %ecx
1188         #xgetbv
1189         .byte   0x0f,0x01,0xd0
1190         shlq    $32, %rdx
1191         orq     %rdx, %rax
1192         ret
1193         SET_SIZE(get_xcr)
1194 
1195         ENTRY(set_xcr)
1196         movq    %rsi, %rdx
1197         shrq    $32, %rdx
1198         movl    %esi, %eax
1199         movl    %edi, %ecx
1200         #xsetbv
1201         .byte   0x0f,0x01,0xd1
1202         ret
1203         SET_SIZE(set_xcr)
1204 
1205         ENTRY(invalidate_cache)
1206         wbinvd
1207         ret
1208         SET_SIZE(invalidate_cache)
1209 
1210         ENTRY_NP(getcregs)
1211 #if defined(__xpv)
1212         /*
1213          * Only a few of the hardware control registers or descriptor tables
1214          * are directly accessible to us, so just zero the structure.
1215          *
1216          * XXPV Perhaps it would be helpful for the hypervisor to return
1217          *      virtualized versions of these for post-mortem use.
1218          *      (Need to reevaluate - perhaps it already does!)
1219          */
1220         pushq   %rdi            /* save *crp */
1221         movq    $CREGSZ, %rsi
1222         call    bzero
1223         popq    %rdi
1224 
1225         /*
1226          * Dump what limited information we can
1227          */
1228         movq    %cr0, %rax
1229         movq    %rax, CREG_CR0(%rdi)    /* cr0 */
1230         movq    %cr2, %rax
1231         movq    %rax, CREG_CR2(%rdi)    /* cr2 */
1232         movq    %cr3, %rax
1233         movq    %rax, CREG_CR3(%rdi)    /* cr3 */
1234         movq    %cr4, %rax
1235         movq    %rax, CREG_CR4(%rdi)    /* cr4 */
1236 
1237 #else   /* __xpv */
1238 
1239 #define GETMSR(r, off, d)       \
1240         movl    $r, %ecx;       \
1241         rdmsr;                  \
1242         movl    %eax, off(d);   \
1243         movl    %edx, off+4(d)
1244 
1245         xorl    %eax, %eax
1246         movq    %rax, CREG_GDT+8(%rdi)
1247         sgdt    CREG_GDT(%rdi)          /* 10 bytes */
1248         movq    %rax, CREG_IDT+8(%rdi)
1249         sidt    CREG_IDT(%rdi)          /* 10 bytes */
1250         movq    %rax, CREG_LDT(%rdi)
1251         sldt    CREG_LDT(%rdi)          /* 2 bytes */
1252         movq    %rax, CREG_TASKR(%rdi)
1253         str     CREG_TASKR(%rdi)        /* 2 bytes */
1254         movq    %cr0, %rax
1255         movq    %rax, CREG_CR0(%rdi)    /* cr0 */
1256         movq    %cr2, %rax
1257         movq    %rax, CREG_CR2(%rdi)    /* cr2 */
1258         movq    %cr3, %rax
1259         movq    %rax, CREG_CR3(%rdi)    /* cr3 */
1260         movq    %cr4, %rax
1261         movq    %rax, CREG_CR4(%rdi)    /* cr4 */
1262         movq    %cr8, %rax
1263         movq    %rax, CREG_CR8(%rdi)    /* cr8 */
1264         GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
1265         GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
1266 #endif  /* __xpv */
1267         ret
1268         SET_SIZE(getcregs)
1269 
1270 #undef GETMSR
1271 
1272 
1273 /*
1274  * A panic trigger is a word which is updated atomically and can only be set
1275  * once.  We atomically store 0xDEFACEDD and load the old value.  If the
1276  * previous value was 0, we succeed and return 1; otherwise return 0.
1277  * This allows a partially corrupt trigger to still trigger correctly.  DTrace
1278  * has its own version of this function to allow it to panic correctly from
1279  * probe context.
1280  */
1281 
1282         ENTRY_NP(panic_trigger)
1283         xorl    %eax, %eax
1284         movl    $0xdefacedd, %edx
1285         lock
1286           xchgl %edx, (%rdi)
1287         cmpl    $0, %edx
1288         je      0f
1289         movl    $0, %eax
1290         ret
1291 0:      movl    $1, %eax
1292         ret
1293         SET_SIZE(panic_trigger)
1294 
1295         ENTRY_NP(dtrace_panic_trigger)
1296         xorl    %eax, %eax
1297         movl    $0xdefacedd, %edx
1298         lock
1299           xchgl %edx, (%rdi)
1300         cmpl    $0, %edx
1301         je      0f
1302         movl    $0, %eax
1303         ret
1304 0:      movl    $1, %eax
1305         ret
1306         SET_SIZE(dtrace_panic_trigger)
1307 
1308 /*
1309  * The panic() and cmn_err() functions invoke vpanic() as a common entry point
1310  * into the panic code implemented in panicsys().  vpanic() is responsible
1311  * for passing through the format string and arguments, and constructing a
1312  * regs structure on the stack into which it saves the current register
1313  * values.  If we are not dying due to a fatal trap, these registers will
1314  * then be preserved in panicbuf as the current processor state.  Before
1315  * invoking panicsys(), vpanic() activates the first panic trigger (see
1316  * common/os/panic.c) and switches to the panic_stack if successful.  Note that
1317  * DTrace takes a slightly different panic path if it must panic from probe
1318  * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
1319  * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
1320  * branches back into vpanic().
1321  */
1322 
1323         ENTRY_NP(vpanic)                        /* Initial stack layout: */
1324 
1325         pushq   %rbp                            /* | %rip |     0x60    */
1326         movq    %rsp, %rbp                      /* | %rbp |     0x58    */
1327         pushfq                                  /* | rfl  |     0x50    */
1328         pushq   %r11                            /* | %r11 |     0x48    */
1329         pushq   %r10                            /* | %r10 |     0x40    */
1330         pushq   %rbx                            /* | %rbx |     0x38    */
1331         pushq   %rax                            /* | %rax |     0x30    */
1332         pushq   %r9                             /* | %r9  |     0x28    */
1333         pushq   %r8                             /* | %r8  |     0x20    */
1334         pushq   %rcx                            /* | %rcx |     0x18    */
1335         pushq   %rdx                            /* | %rdx |     0x10    */
1336         pushq   %rsi                            /* | %rsi |     0x8 alist */
1337         pushq   %rdi                            /* | %rdi |     0x0 format */
1338 
1339         movq    %rsp, %rbx                      /* %rbx = current %rsp */
1340 
1341         leaq    panic_quiesce(%rip), %rdi       /* %rdi = &panic_quiesce */
1342         call    panic_trigger                   /* %eax = panic_trigger() */
1343 
1344 vpanic_common:
1345         /*
1346          * The panic_trigger result is in %eax from the call above, and
1347          * dtrace_panic places it in %eax before branching here.
1348          * The rdmsr instructions that follow below will clobber %eax so
1349          * we stash the panic_trigger result in %r11d.
1350          */
1351         movl    %eax, %r11d
1352         cmpl    $0, %r11d
1353         je      0f
1354 
1355         /*
1356          * If panic_trigger() was successful, we are the first to initiate a
1357          * panic: we now switch to the reserved panic_stack before continuing.
1358          */
1359         leaq    panic_stack(%rip), %rsp
1360         addq    $PANICSTKSIZE, %rsp
1361 0:      subq    $REGSIZE, %rsp
1362         /*
1363          * Now that we've got everything set up, store the register values as
1364          * they were when we entered vpanic() to the designated location in
1365          * the regs structure we allocated on the stack.
1366          */
1367         movq    0x0(%rbx), %rcx
1368         movq    %rcx, REGOFF_RDI(%rsp)
1369         movq    0x8(%rbx), %rcx
1370         movq    %rcx, REGOFF_RSI(%rsp)
1371         movq    0x10(%rbx), %rcx
1372         movq    %rcx, REGOFF_RDX(%rsp)
1373         movq    0x18(%rbx), %rcx
1374         movq    %rcx, REGOFF_RCX(%rsp)
1375         movq    0x20(%rbx), %rcx
1376 
1377         movq    %rcx, REGOFF_R8(%rsp)
1378         movq    0x28(%rbx), %rcx
1379         movq    %rcx, REGOFF_R9(%rsp)
1380         movq    0x30(%rbx), %rcx
1381         movq    %rcx, REGOFF_RAX(%rsp)
1382         movq    0x38(%rbx), %rcx
1383         movq    %rcx, REGOFF_RBX(%rsp)
1384         movq    0x58(%rbx), %rcx
1385 
1386         movq    %rcx, REGOFF_RBP(%rsp)
1387         movq    0x40(%rbx), %rcx
1388         movq    %rcx, REGOFF_R10(%rsp)
1389         movq    0x48(%rbx), %rcx
1390         movq    %rcx, REGOFF_R11(%rsp)
1391         movq    %r12, REGOFF_R12(%rsp)
1392 
1393         movq    %r13, REGOFF_R13(%rsp)
1394         movq    %r14, REGOFF_R14(%rsp)
1395         movq    %r15, REGOFF_R15(%rsp)
1396 
1397         xorl    %ecx, %ecx
1398         movw    %ds, %cx
1399         movq    %rcx, REGOFF_DS(%rsp)
1400         movw    %es, %cx
1401         movq    %rcx, REGOFF_ES(%rsp)
1402         movw    %fs, %cx
1403         movq    %rcx, REGOFF_FS(%rsp)
1404         movw    %gs, %cx
1405         movq    %rcx, REGOFF_GS(%rsp)
1406 
1407         movq    $0, REGOFF_TRAPNO(%rsp)
1408 
1409         movq    $0, REGOFF_ERR(%rsp)
1410         leaq    vpanic(%rip), %rcx
1411         movq    %rcx, REGOFF_RIP(%rsp)
1412         movw    %cs, %cx
1413         movzwq  %cx, %rcx
1414         movq    %rcx, REGOFF_CS(%rsp)
1415         movq    0x50(%rbx), %rcx
1416         movq    %rcx, REGOFF_RFL(%rsp)
1417         movq    %rbx, %rcx
1418         addq    $0x60, %rcx
1419         movq    %rcx, REGOFF_RSP(%rsp)
1420         movw    %ss, %cx
1421         movzwq  %cx, %rcx
1422         movq    %rcx, REGOFF_SS(%rsp)
1423 
1424         /*
1425          * panicsys(format, alist, rp, on_panic_stack)
1426          */
1427         movq    REGOFF_RDI(%rsp), %rdi          /* format */
1428         movq    REGOFF_RSI(%rsp), %rsi          /* alist */
1429         movq    %rsp, %rdx                      /* struct regs */
1430         movl    %r11d, %ecx                     /* on_panic_stack */
1431         call    panicsys
1432         addq    $REGSIZE, %rsp
1433         popq    %rdi
1434         popq    %rsi
1435         popq    %rdx
1436         popq    %rcx
1437         popq    %r8
1438         popq    %r9
1439         popq    %rax
1440         popq    %rbx
1441         popq    %r10
1442         popq    %r11
1443         popfq
1444         leave
1445         ret
1446         SET_SIZE(vpanic)
1447 
1448         ENTRY_NP(dtrace_vpanic)                 /* Initial stack layout: */
1449 
1450         pushq   %rbp                            /* | %rip |     0x60    */
1451         movq    %rsp, %rbp                      /* | %rbp |     0x58    */
1452         pushfq                                  /* | rfl  |     0x50    */
1453         pushq   %r11                            /* | %r11 |     0x48    */
1454         pushq   %r10                            /* | %r10 |     0x40    */
1455         pushq   %rbx                            /* | %rbx |     0x38    */
1456         pushq   %rax                            /* | %rax |     0x30    */
1457         pushq   %r9                             /* | %r9  |     0x28    */
1458         pushq   %r8                             /* | %r8  |     0x20    */
1459         pushq   %rcx                            /* | %rcx |     0x18    */
1460         pushq   %rdx                            /* | %rdx |     0x10    */
1461         pushq   %rsi                            /* | %rsi |     0x8 alist */
1462         pushq   %rdi                            /* | %rdi |     0x0 format */
1463 
1464         movq    %rsp, %rbx                      /* %rbx = current %rsp */
1465 
1466         leaq    panic_quiesce(%rip), %rdi       /* %rdi = &panic_quiesce */
1467         call    dtrace_panic_trigger    /* %eax = dtrace_panic_trigger() */
1468         jmp     vpanic_common
1469 
1470         SET_SIZE(dtrace_vpanic)
1471 
1472         DGDEF3(timedelta, 8, 8)
1473         .long   0, 0
1474 
1475         /*
1476          * initialized to a non zero value to make pc_gethrtime()
1477          * work correctly even before clock is initialized
1478          */
1479         DGDEF3(hrtime_base, 8, 8)
1480         .long   _MUL(NSEC_PER_CLOCK_TICK, 6), 0
1481 
1482         DGDEF3(adj_shift, 4, 4)
1483         .long   ADJ_SHIFT
1484 
1485         ENTRY_NP(hres_tick)
1486         pushq   %rbp
1487         movq    %rsp, %rbp
1488 
1489         /*
1490          * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
1491          * hres_last_tick can only be modified while holding CLOCK_LOCK).
1492          * At worst, performing this now instead of under CLOCK_LOCK may
1493          * introduce some jitter in pc_gethrestime().
1494          */
1495         movq    gethrtimef(%rip), %rsi
1496         INDIRECT_CALL_REG(rsi)
1497         movq    %rax, %r8
1498 
1499         leaq    hres_lock(%rip), %rax
1500         movb    $-1, %dl
1501 .CL1:
1502         xchgb   %dl, (%rax)
1503         testb   %dl, %dl
1504         jz      .CL3                    /* got it */
1505 .CL2:
1506         cmpb    $0, (%rax)              /* possible to get lock? */
1507         pause
1508         jne     .CL2
1509         jmp     .CL1                    /* yes, try again */
1510 .CL3:
1511         /*
1512          * compute the interval since last time hres_tick was called
1513          * and adjust hrtime_base and hrestime accordingly
1514          * hrtime_base is an 8 byte value (in nsec), hrestime is
1515          * a timestruc_t (sec, nsec)
1516          */
1517         leaq    hres_last_tick(%rip), %rax
1518         movq    %r8, %r11
1519         subq    (%rax), %r8
1520         addq    %r8, hrtime_base(%rip)  /* add interval to hrtime_base */
1521         addq    %r8, hrestime+8(%rip)   /* add interval to hrestime.tv_nsec */
1522         /*
1523          * Now that we have CLOCK_LOCK, we can update hres_last_tick
1524          */
1525         movq    %r11, (%rax)
1526 
1527         call    __adj_hrestime
1528 
1529         /*
1530          * release the hres_lock
1531          */
1532         incl    hres_lock(%rip)
1533         leave
1534         ret
1535         SET_SIZE(hres_tick)
1536 
1537 /*
1538  * void prefetch_smap_w(void *)
1539  *
1540  * Prefetch ahead within a linear list of smap structures.
1541  * Not implemented for ia32.  Stub for compatibility.
1542  */
1543 
1544         ENTRY(prefetch_smap_w)
1545         rep;    ret     /* use 2 byte return instruction when branch target */
1546                         /* AMD Software Optimization Guide - Section 6.2 */
1547         SET_SIZE(prefetch_smap_w)
1548 
1549 /*
1550  * prefetch_page_r(page_t *)
1551  * issue prefetch instructions for a page_t
1552  */
1553 
1554         ENTRY(prefetch_page_r)
1555         rep;    ret     /* use 2 byte return instruction when branch target */
1556                         /* AMD Software Optimization Guide - Section 6.2 */
1557         SET_SIZE(prefetch_page_r)
1558 
1559         ENTRY(bcmp)
1560         pushq   %rbp
1561         movq    %rsp, %rbp
1562 #ifdef DEBUG
1563         testq   %rdx,%rdx
1564         je      1f
1565         movq    postbootkernelbase(%rip), %r11
1566         cmpq    %r11, %rdi
1567         jb      0f
1568         cmpq    %r11, %rsi
1569         jnb     1f
1570 0:      leaq    .bcmp_panic_msg(%rip), %rdi
1571         xorl    %eax, %eax
1572         call    panic
1573 1:
1574 #endif  /* DEBUG */
1575         call    memcmp
1576         testl   %eax, %eax
1577         setne   %dl
1578         leave
1579         movzbl  %dl, %eax
1580         ret
1581         SET_SIZE(bcmp)
1582 
1583 #ifdef DEBUG
1584         .text
1585 .bcmp_panic_msg:
1586         .string "bcmp: arguments below kernelbase"
1587 #endif  /* DEBUG */
1588 
1589         ENTRY_NP(bsrw_insn)
1590         xorl    %eax, %eax
1591         bsrw    %di, %ax
1592         ret
1593         SET_SIZE(bsrw_insn)
1594 
1595         ENTRY_NP(switch_sp_and_call)
1596         pushq   %rbp
1597         movq    %rsp, %rbp              /* set up stack frame */
1598         movq    %rdi, %rsp              /* switch stack pointer */
1599         movq    %rdx, %rdi              /* pass func arg 1 */
1600         movq    %rsi, %r11              /* save function to call */
1601         movq    %rcx, %rsi              /* pass func arg 2 */
1602         INDIRECT_CALL_REG(r11)          /* call function */
1603         leave                           /* restore stack */
1604         ret
1605         SET_SIZE(switch_sp_and_call)
1606 
1607         ENTRY_NP(kmdb_enter)
1608         pushq   %rbp
1609         movq    %rsp, %rbp
1610 
1611         /*
1612          * Save flags, do a 'cli' then return the saved flags
1613          */
1614         call    intr_clear
1615 
1616         int     $T_DBGENTR
1617 
1618         /*
1619          * Restore the saved flags
1620          */
1621         movq    %rax, %rdi
1622         call    intr_restore
1623 
1624         leave
1625         ret
1626         SET_SIZE(kmdb_enter)
1627 
1628         ENTRY_NP(return_instr)
1629         rep;    ret     /* use 2 byte instruction when branch target */
1630                         /* AMD Software Optimization Guide - Section 6.2 */
1631         SET_SIZE(return_instr)
1632 
1633         ENTRY(getflags)
1634         pushfq
1635         popq    %rax
1636 #if defined(__xpv)
1637         CURTHREAD(%rdi)
1638         KPREEMPT_DISABLE(%rdi)
1639         /*
1640          * Synthesize the PS_IE bit from the event mask bit
1641          */
1642         CURVCPU(%r11)
1643         andq    $_BITNOT(PS_IE), %rax
1644         XEN_TEST_UPCALL_MASK(%r11)
1645         jnz     1f
1646         orq     $PS_IE, %rax
1647 1:
1648         KPREEMPT_ENABLE_NOKP(%rdi)
1649 #endif
1650         ret
1651         SET_SIZE(getflags)
1652 
1653         ENTRY(ftrace_interrupt_disable)
1654         pushfq
1655         popq    %rax
1656         CLI(%rdx)
1657         ret
1658         SET_SIZE(ftrace_interrupt_disable)
1659 
1660         ENTRY(ftrace_interrupt_enable)
1661         pushq   %rdi
1662         popfq
1663         ret
1664         SET_SIZE(ftrace_interrupt_enable)
1665 
1666         ENTRY(clflush_insn)
1667         clflush (%rdi)
1668         ret
1669         SET_SIZE(clflush_insn)
1670 
1671         ENTRY(mfence_insn)
1672         mfence
1673         ret
1674         SET_SIZE(mfence_insn)
1675 
1676 /*
1677  * VMware implements an I/O port that programs can query to detect if software
1678  * is running in a VMware hypervisor. This hypervisor port behaves differently
1679  * depending on magic values in certain registers and modifies some registers
1680  * as a side effect.
1681  *
1682  * References: http://kb.vmware.com/kb/1009458
1683  */
1684 
1685         ENTRY(vmware_port)
1686         pushq   %rbx
1687         movl    $VMWARE_HVMAGIC, %eax
1688         movl    $0xffffffff, %ebx
1689         movl    %edi, %ecx
1690         movl    $VMWARE_HVPORT, %edx
1691         inl     (%dx)
1692         movl    %eax, (%rsi)
1693         movl    %ebx, 4(%rsi)
1694         movl    %ecx, 8(%rsi)
1695         movl    %edx, 12(%rsi)
1696         popq    %rbx
1697         ret
1698         SET_SIZE(vmware_port)
1699