1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2014 by Delphix. All rights reserved. 26 * Copyright 2016 Joyent, Inc. 27 */ 28 29 /* 30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 32 * All Rights Reserved 33 */ 34 35 /* 36 * Copyright (c) 2009, Intel Corporation. 37 * All rights reserved. 38 */ 39 40 /* 41 * General assembly language routines. 42 * It is the intent of this file to contain routines that are 43 * independent of the specific kernel architecture, and those that are 44 * common across kernel architectures. 45 * As architectures diverge, and implementations of specific 46 * architecture-dependent routines change, the routines should be moved 47 * from this file into the respective ../`arch -k`/subr.s file. 48 */ 49 50 #include <sys/asm_linkage.h> 51 #include <sys/asm_misc.h> 52 #include <sys/panic.h> 53 #include <sys/ontrap.h> 54 #include <sys/regset.h> 55 #include <sys/privregs.h> 56 #include <sys/reboot.h> 57 #include <sys/psw.h> 58 #include <sys/x86_archext.h> 59 60 #if defined(__lint) 61 #include <sys/types.h> 62 #include <sys/systm.h> 63 #include <sys/thread.h> 64 #include <sys/archsystm.h> 65 #include <sys/byteorder.h> 66 #include <sys/dtrace.h> 67 #include <sys/ftrace.h> 68 #else /* __lint */ 69 #include "assym.h" 70 #endif /* __lint */ 71 #include <sys/dditypes.h> 72 73 /* 74 * on_fault() 75 * 76 * Catch lofault faults. Like setjmp except it returns one 77 * if code following causes uncorrectable fault. Turned off 78 * by calling no_fault(). Note that while under on_fault(), 79 * SMAP is disabled. For more information see 80 * uts/intel/ia32/ml/copy.s. 81 */ 82 83 #if defined(__lint) 84 85 /* ARGSUSED */ 86 int 87 on_fault(label_t *ljb) 88 { return (0); } 89 90 void 91 no_fault(void) 92 {} 93 94 #else /* __lint */ 95 96 #if defined(__amd64) 97 98 ENTRY(on_fault) 99 movq %gs:CPU_THREAD, %rsi 100 leaq catch_fault(%rip), %rdx 101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 103 call smap_disable /* allow user accesses */ 104 jmp setjmp /* let setjmp do the rest */ 105 106 catch_fault: 107 movq %gs:CPU_THREAD, %rsi 108 movq T_ONFAULT(%rsi), %rdi /* address of save area */ 109 xorl %eax, %eax 110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 112 call smap_enable /* disallow user accesses */ 113 jmp longjmp /* let longjmp do the rest */ 114 SET_SIZE(on_fault) 115 116 ENTRY(no_fault) 117 movq %gs:CPU_THREAD, %rsi 118 xorl %eax, %eax 119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 121 call smap_enable /* disallow user accesses */ 122 ret 123 SET_SIZE(no_fault) 124 125 #elif defined(__i386) 126 127 ENTRY(on_fault) 128 movl %gs:CPU_THREAD, %edx 129 movl 4(%esp), %eax /* jumpbuf address */ 130 leal catch_fault, %ecx 131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */ 132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */ 133 jmp setjmp /* let setjmp do the rest */ 134 135 catch_fault: 136 movl %gs:CPU_THREAD, %edx 137 xorl %eax, %eax 138 movl T_ONFAULT(%edx), %ecx /* address of save area */ 139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 141 pushl %ecx 142 call longjmp /* let longjmp do the rest */ 143 SET_SIZE(on_fault) 144 145 ENTRY(no_fault) 146 movl %gs:CPU_THREAD, %edx 147 xorl %eax, %eax 148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 150 ret 151 SET_SIZE(no_fault) 152 153 #endif /* __i386 */ 154 #endif /* __lint */ 155 156 /* 157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 159 */ 160 161 #if defined(lint) 162 163 void 164 on_trap_trampoline(void) 165 {} 166 167 #else /* __lint */ 168 169 #if defined(__amd64) 170 171 ENTRY(on_trap_trampoline) 172 movq %gs:CPU_THREAD, %rsi 173 movq T_ONTRAP(%rsi), %rdi 174 addq $OT_JMPBUF, %rdi 175 jmp longjmp 176 SET_SIZE(on_trap_trampoline) 177 178 #elif defined(__i386) 179 180 ENTRY(on_trap_trampoline) 181 movl %gs:CPU_THREAD, %eax 182 movl T_ONTRAP(%eax), %eax 183 addl $OT_JMPBUF, %eax 184 pushl %eax 185 call longjmp 186 SET_SIZE(on_trap_trampoline) 187 188 #endif /* __i386 */ 189 #endif /* __lint */ 190 191 /* 192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 193 * more information about the on_trap() mechanism. If the on_trap_data is the 194 * same as the topmost stack element, we just modify that element. 195 */ 196 #if defined(lint) 197 198 /*ARGSUSED*/ 199 int 200 on_trap(on_trap_data_t *otp, uint_t prot) 201 { return (0); } 202 203 #else /* __lint */ 204 205 #if defined(__amd64) 206 207 ENTRY(on_trap) 208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 212 xorl %ecx, %ecx 213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 217 cmpq %rdi, %rcx /* if (otp == %rcx) */ 218 je 0f /* don't modify t_ontrap */ 219 220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 222 223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 224 jmp setjmp 225 SET_SIZE(on_trap) 226 227 #elif defined(__i386) 228 229 ENTRY(on_trap) 230 movl 4(%esp), %eax /* %eax = otp */ 231 movl 8(%esp), %edx /* %edx = prot */ 232 233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */ 234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */ 235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */ 236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */ 237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */ 238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */ 239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */ 240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */ 241 cmpl %eax, %ecx /* if (otp == %ecx) */ 242 je 0f /* don't modify t_ontrap */ 243 244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */ 245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */ 246 247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */ 248 movl %eax, 4(%esp) /* put %eax back on the stack */ 249 jmp setjmp /* let setjmp do the rest */ 250 SET_SIZE(on_trap) 251 252 #endif /* __i386 */ 253 #endif /* __lint */ 254 255 /* 256 * Setjmp and longjmp implement non-local gotos using state vectors 257 * type label_t. 258 */ 259 260 #if defined(__lint) 261 262 /* ARGSUSED */ 263 int 264 setjmp(label_t *lp) 265 { return (0); } 266 267 /* ARGSUSED */ 268 void 269 longjmp(label_t *lp) 270 {} 271 272 #else /* __lint */ 273 274 #if LABEL_PC != 0 275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 276 #endif /* LABEL_PC != 0 */ 277 278 #if defined(__amd64) 279 280 ENTRY(setjmp) 281 movq %rsp, LABEL_SP(%rdi) 282 movq %rbp, LABEL_RBP(%rdi) 283 movq %rbx, LABEL_RBX(%rdi) 284 movq %r12, LABEL_R12(%rdi) 285 movq %r13, LABEL_R13(%rdi) 286 movq %r14, LABEL_R14(%rdi) 287 movq %r15, LABEL_R15(%rdi) 288 movq (%rsp), %rdx /* return address */ 289 movq %rdx, (%rdi) /* LABEL_PC is 0 */ 290 xorl %eax, %eax /* return 0 */ 291 ret 292 SET_SIZE(setjmp) 293 294 ENTRY(longjmp) 295 movq LABEL_SP(%rdi), %rsp 296 movq LABEL_RBP(%rdi), %rbp 297 movq LABEL_RBX(%rdi), %rbx 298 movq LABEL_R12(%rdi), %r12 299 movq LABEL_R13(%rdi), %r13 300 movq LABEL_R14(%rdi), %r14 301 movq LABEL_R15(%rdi), %r15 302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 303 movq %rdx, (%rsp) 304 xorl %eax, %eax 305 incl %eax /* return 1 */ 306 ret 307 SET_SIZE(longjmp) 308 309 #elif defined(__i386) 310 311 ENTRY(setjmp) 312 movl 4(%esp), %edx /* address of save area */ 313 movl %ebp, LABEL_EBP(%edx) 314 movl %ebx, LABEL_EBX(%edx) 315 movl %esi, LABEL_ESI(%edx) 316 movl %edi, LABEL_EDI(%edx) 317 movl %esp, 4(%edx) 318 movl (%esp), %ecx /* %eip (return address) */ 319 movl %ecx, (%edx) /* LABEL_PC is 0 */ 320 subl %eax, %eax /* return 0 */ 321 ret 322 SET_SIZE(setjmp) 323 324 ENTRY(longjmp) 325 movl 4(%esp), %edx /* address of save area */ 326 movl LABEL_EBP(%edx), %ebp 327 movl LABEL_EBX(%edx), %ebx 328 movl LABEL_ESI(%edx), %esi 329 movl LABEL_EDI(%edx), %edi 330 movl 4(%edx), %esp 331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */ 332 movl $1, %eax 333 addl $4, %esp /* pop ret adr */ 334 jmp *%ecx /* indirect */ 335 SET_SIZE(longjmp) 336 337 #endif /* __i386 */ 338 #endif /* __lint */ 339 340 /* 341 * if a() calls b() calls caller(), 342 * caller() returns return address in a(). 343 * (Note: We assume a() and b() are C routines which do the normal entry/exit 344 * sequence.) 345 */ 346 347 #if defined(__lint) 348 349 caddr_t 350 caller(void) 351 { return (0); } 352 353 #else /* __lint */ 354 355 #if defined(__amd64) 356 357 ENTRY(caller) 358 movq 8(%rbp), %rax /* b()'s return pc, in a() */ 359 ret 360 SET_SIZE(caller) 361 362 #elif defined(__i386) 363 364 ENTRY(caller) 365 movl 4(%ebp), %eax /* b()'s return pc, in a() */ 366 ret 367 SET_SIZE(caller) 368 369 #endif /* __i386 */ 370 #endif /* __lint */ 371 372 /* 373 * if a() calls callee(), callee() returns the 374 * return address in a(); 375 */ 376 377 #if defined(__lint) 378 379 caddr_t 380 callee(void) 381 { return (0); } 382 383 #else /* __lint */ 384 385 #if defined(__amd64) 386 387 ENTRY(callee) 388 movq (%rsp), %rax /* callee()'s return pc, in a() */ 389 ret 390 SET_SIZE(callee) 391 392 #elif defined(__i386) 393 394 ENTRY(callee) 395 movl (%esp), %eax /* callee()'s return pc, in a() */ 396 ret 397 SET_SIZE(callee) 398 399 #endif /* __i386 */ 400 #endif /* __lint */ 401 402 /* 403 * return the current frame pointer 404 */ 405 406 #if defined(__lint) 407 408 greg_t 409 getfp(void) 410 { return (0); } 411 412 #else /* __lint */ 413 414 #if defined(__amd64) 415 416 ENTRY(getfp) 417 movq %rbp, %rax 418 ret 419 SET_SIZE(getfp) 420 421 #elif defined(__i386) 422 423 ENTRY(getfp) 424 movl %ebp, %eax 425 ret 426 SET_SIZE(getfp) 427 428 #endif /* __i386 */ 429 #endif /* __lint */ 430 431 /* 432 * Invalidate a single page table entry in the TLB 433 */ 434 435 #if defined(__lint) 436 437 /* ARGSUSED */ 438 void 439 mmu_tlbflush_entry(caddr_t m) 440 {} 441 442 #else /* __lint */ 443 444 #if defined(__amd64) 445 446 ENTRY(mmu_tlbflush_entry) 447 invlpg (%rdi) 448 ret 449 SET_SIZE(mmu_tlbflush_entry) 450 451 #elif defined(__i386) 452 453 ENTRY(mmu_tlbflush_entry) 454 movl 4(%esp), %eax 455 invlpg (%eax) 456 ret 457 SET_SIZE(mmu_tlbflush_entry) 458 459 #endif /* __i386 */ 460 #endif /* __lint */ 461 462 463 /* 464 * Get/Set the value of various control registers 465 */ 466 467 #if defined(__lint) 468 469 ulong_t 470 getcr0(void) 471 { return (0); } 472 473 /* ARGSUSED */ 474 void 475 setcr0(ulong_t value) 476 {} 477 478 ulong_t 479 getcr2(void) 480 { return (0); } 481 482 ulong_t 483 getcr3(void) 484 { return (0); } 485 486 #if !defined(__xpv) 487 /* ARGSUSED */ 488 void 489 setcr3(ulong_t val) 490 {} 491 492 void 493 reload_cr3(void) 494 {} 495 #endif 496 497 ulong_t 498 getcr4(void) 499 { return (0); } 500 501 /* ARGSUSED */ 502 void 503 setcr4(ulong_t val) 504 {} 505 506 #if defined(__amd64) 507 508 ulong_t 509 getcr8(void) 510 { return (0); } 511 512 /* ARGSUSED */ 513 void 514 setcr8(ulong_t val) 515 {} 516 517 #endif /* __amd64 */ 518 519 #else /* __lint */ 520 521 #if defined(__amd64) 522 523 ENTRY(getcr0) 524 movq %cr0, %rax 525 ret 526 SET_SIZE(getcr0) 527 528 ENTRY(setcr0) 529 movq %rdi, %cr0 530 ret 531 SET_SIZE(setcr0) 532 533 ENTRY(getcr2) 534 #if defined(__xpv) 535 movq %gs:CPU_VCPU_INFO, %rax 536 movq VCPU_INFO_ARCH_CR2(%rax), %rax 537 #else 538 movq %cr2, %rax 539 #endif 540 ret 541 SET_SIZE(getcr2) 542 543 ENTRY(getcr3) 544 movq %cr3, %rax 545 ret 546 SET_SIZE(getcr3) 547 548 #if !defined(__xpv) 549 550 ENTRY(setcr3) 551 movq %rdi, %cr3 552 ret 553 SET_SIZE(setcr3) 554 555 ENTRY(reload_cr3) 556 movq %cr3, %rdi 557 movq %rdi, %cr3 558 ret 559 SET_SIZE(reload_cr3) 560 561 #endif /* __xpv */ 562 563 ENTRY(getcr4) 564 movq %cr4, %rax 565 ret 566 SET_SIZE(getcr4) 567 568 ENTRY(setcr4) 569 movq %rdi, %cr4 570 ret 571 SET_SIZE(setcr4) 572 573 ENTRY(getcr8) 574 movq %cr8, %rax 575 ret 576 SET_SIZE(getcr8) 577 578 ENTRY(setcr8) 579 movq %rdi, %cr8 580 ret 581 SET_SIZE(setcr8) 582 583 #elif defined(__i386) 584 585 ENTRY(getcr0) 586 movl %cr0, %eax 587 ret 588 SET_SIZE(getcr0) 589 590 ENTRY(setcr0) 591 movl 4(%esp), %eax 592 movl %eax, %cr0 593 ret 594 SET_SIZE(setcr0) 595 596 /* 597 * "lock mov %cr0" is used on processors which indicate it is 598 * supported via CPUID. Normally the 32 bit TPR is accessed via 599 * the local APIC. 600 */ 601 ENTRY(getcr8) 602 lock 603 movl %cr0, %eax 604 ret 605 SET_SIZE(getcr8) 606 607 ENTRY(setcr8) 608 movl 4(%esp), %eax 609 lock 610 movl %eax, %cr0 611 ret 612 SET_SIZE(setcr8) 613 614 ENTRY(getcr2) 615 #if defined(__xpv) 616 movl %gs:CPU_VCPU_INFO, %eax 617 movl VCPU_INFO_ARCH_CR2(%eax), %eax 618 #else 619 movl %cr2, %eax 620 #endif 621 ret 622 SET_SIZE(getcr2) 623 624 ENTRY(getcr3) 625 movl %cr3, %eax 626 ret 627 SET_SIZE(getcr3) 628 629 #if !defined(__xpv) 630 631 ENTRY(setcr3) 632 movl 4(%esp), %eax 633 movl %eax, %cr3 634 ret 635 SET_SIZE(setcr3) 636 637 ENTRY(reload_cr3) 638 movl %cr3, %eax 639 movl %eax, %cr3 640 ret 641 SET_SIZE(reload_cr3) 642 643 #endif /* __xpv */ 644 645 ENTRY(getcr4) 646 movl %cr4, %eax 647 ret 648 SET_SIZE(getcr4) 649 650 ENTRY(setcr4) 651 movl 4(%esp), %eax 652 movl %eax, %cr4 653 ret 654 SET_SIZE(setcr4) 655 656 #endif /* __i386 */ 657 #endif /* __lint */ 658 659 #if defined(__lint) 660 661 /*ARGSUSED*/ 662 uint32_t 663 __cpuid_insn(struct cpuid_regs *regs) 664 { return (0); } 665 666 #else /* __lint */ 667 668 #if defined(__amd64) 669 670 ENTRY(__cpuid_insn) 671 movq %rbx, %r8 672 movq %rcx, %r9 673 movq %rdx, %r11 674 movl (%rdi), %eax /* %eax = regs->cp_eax */ 675 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 676 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 677 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 678 cpuid 679 movl %eax, (%rdi) /* regs->cp_eax = %eax */ 680 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 681 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 682 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 683 movq %r8, %rbx 684 movq %r9, %rcx 685 movq %r11, %rdx 686 ret 687 SET_SIZE(__cpuid_insn) 688 689 #elif defined(__i386) 690 691 ENTRY(__cpuid_insn) 692 pushl %ebp 693 movl 0x8(%esp), %ebp /* %ebp = regs */ 694 pushl %ebx 695 pushl %ecx 696 pushl %edx 697 movl (%ebp), %eax /* %eax = regs->cp_eax */ 698 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */ 699 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */ 700 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */ 701 cpuid 702 movl %eax, (%ebp) /* regs->cp_eax = %eax */ 703 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */ 704 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */ 705 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */ 706 popl %edx 707 popl %ecx 708 popl %ebx 709 popl %ebp 710 ret 711 SET_SIZE(__cpuid_insn) 712 713 #endif /* __i386 */ 714 #endif /* __lint */ 715 716 #if defined(__lint) 717 718 /*ARGSUSED*/ 719 void 720 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints) 721 {} 722 723 #else /* __lint */ 724 725 #if defined(__amd64) 726 727 ENTRY_NP(i86_monitor) 728 pushq %rbp 729 movq %rsp, %rbp 730 movq %rdi, %rax /* addr */ 731 movq %rsi, %rcx /* extensions */ 732 /* rdx contains input arg3: hints */ 733 clflush (%rax) 734 .byte 0x0f, 0x01, 0xc8 /* monitor */ 735 leave 736 ret 737 SET_SIZE(i86_monitor) 738 739 #elif defined(__i386) 740 741 ENTRY_NP(i86_monitor) 742 pushl %ebp 743 movl %esp, %ebp 744 movl 0x8(%ebp),%eax /* addr */ 745 movl 0xc(%ebp),%ecx /* extensions */ 746 movl 0x10(%ebp),%edx /* hints */ 747 clflush (%eax) 748 .byte 0x0f, 0x01, 0xc8 /* monitor */ 749 leave 750 ret 751 SET_SIZE(i86_monitor) 752 753 #endif /* __i386 */ 754 #endif /* __lint */ 755 756 #if defined(__lint) 757 758 /*ARGSUSED*/ 759 void 760 i86_mwait(uint32_t data, uint32_t extensions) 761 {} 762 763 #else /* __lint */ 764 765 #if defined(__amd64) 766 767 ENTRY_NP(i86_mwait) 768 pushq %rbp 769 movq %rsp, %rbp 770 movq %rdi, %rax /* data */ 771 movq %rsi, %rcx /* extensions */ 772 .byte 0x0f, 0x01, 0xc9 /* mwait */ 773 leave 774 ret 775 SET_SIZE(i86_mwait) 776 777 #elif defined(__i386) 778 779 ENTRY_NP(i86_mwait) 780 pushl %ebp 781 movl %esp, %ebp 782 movl 0x8(%ebp),%eax /* data */ 783 movl 0xc(%ebp),%ecx /* extensions */ 784 .byte 0x0f, 0x01, 0xc9 /* mwait */ 785 leave 786 ret 787 SET_SIZE(i86_mwait) 788 789 #endif /* __i386 */ 790 #endif /* __lint */ 791 792 #if defined(__xpv) 793 /* 794 * Defined in C 795 */ 796 #else 797 798 #if defined(__lint) 799 800 hrtime_t 801 tsc_read(void) 802 { 803 return (0); 804 } 805 806 #else /* __lint */ 807 808 #if defined(__amd64) 809 810 ENTRY_NP(tsc_read) 811 movq %rbx, %r11 812 movl $0, %eax 813 cpuid 814 rdtsc 815 movq %r11, %rbx 816 shlq $32, %rdx 817 orq %rdx, %rax 818 ret 819 .globl _tsc_mfence_start 820 _tsc_mfence_start: 821 mfence 822 rdtsc 823 shlq $32, %rdx 824 orq %rdx, %rax 825 ret 826 .globl _tsc_mfence_end 827 _tsc_mfence_end: 828 .globl _tscp_start 829 _tscp_start: 830 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 831 shlq $32, %rdx 832 orq %rdx, %rax 833 ret 834 .globl _tscp_end 835 _tscp_end: 836 .globl _no_rdtsc_start 837 _no_rdtsc_start: 838 xorl %edx, %edx 839 xorl %eax, %eax 840 ret 841 .globl _no_rdtsc_end 842 _no_rdtsc_end: 843 .globl _tsc_lfence_start 844 _tsc_lfence_start: 845 lfence 846 rdtsc 847 shlq $32, %rdx 848 orq %rdx, %rax 849 ret 850 .globl _tsc_lfence_end 851 _tsc_lfence_end: 852 SET_SIZE(tsc_read) 853 854 #else /* __i386 */ 855 856 ENTRY_NP(tsc_read) 857 pushl %ebx 858 movl $0, %eax 859 cpuid 860 rdtsc 861 popl %ebx 862 ret 863 .globl _tsc_mfence_start 864 _tsc_mfence_start: 865 mfence 866 rdtsc 867 ret 868 .globl _tsc_mfence_end 869 _tsc_mfence_end: 870 .globl _tscp_start 871 _tscp_start: 872 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 873 ret 874 .globl _tscp_end 875 _tscp_end: 876 .globl _no_rdtsc_start 877 _no_rdtsc_start: 878 xorl %edx, %edx 879 xorl %eax, %eax 880 ret 881 .globl _no_rdtsc_end 882 _no_rdtsc_end: 883 .globl _tsc_lfence_start 884 _tsc_lfence_start: 885 lfence 886 rdtsc 887 ret 888 .globl _tsc_lfence_end 889 _tsc_lfence_end: 890 SET_SIZE(tsc_read) 891 892 #endif /* __i386 */ 893 894 #endif /* __lint */ 895 896 897 #endif /* __xpv */ 898 899 #ifdef __lint 900 /* 901 * Do not use this function for obtaining clock tick. This 902 * is called by callers who do not need to have a guarenteed 903 * correct tick value. The proper routine to use is tsc_read(). 904 */ 905 u_longlong_t 906 randtick(void) 907 { 908 return (0); 909 } 910 #else 911 #if defined(__amd64) 912 ENTRY_NP(randtick) 913 rdtsc 914 shlq $32, %rdx 915 orq %rdx, %rax 916 ret 917 SET_SIZE(randtick) 918 #else 919 ENTRY_NP(randtick) 920 rdtsc 921 ret 922 SET_SIZE(randtick) 923 #endif /* __i386 */ 924 #endif /* __lint */ 925 /* 926 * Insert entryp after predp in a doubly linked list. 927 */ 928 929 #if defined(__lint) 930 931 /*ARGSUSED*/ 932 void 933 _insque(caddr_t entryp, caddr_t predp) 934 {} 935 936 #else /* __lint */ 937 938 #if defined(__amd64) 939 940 ENTRY(_insque) 941 movq (%rsi), %rax /* predp->forw */ 942 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 943 movq %rax, (%rdi) /* entryp->forw = predp->forw */ 944 movq %rdi, (%rsi) /* predp->forw = entryp */ 945 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 946 ret 947 SET_SIZE(_insque) 948 949 #elif defined(__i386) 950 951 ENTRY(_insque) 952 movl 8(%esp), %edx 953 movl 4(%esp), %ecx 954 movl (%edx), %eax /* predp->forw */ 955 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */ 956 movl %eax, (%ecx) /* entryp->forw = predp->forw */ 957 movl %ecx, (%edx) /* predp->forw = entryp */ 958 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */ 959 ret 960 SET_SIZE(_insque) 961 962 #endif /* __i386 */ 963 #endif /* __lint */ 964 965 /* 966 * Remove entryp from a doubly linked list 967 */ 968 969 #if defined(__lint) 970 971 /*ARGSUSED*/ 972 void 973 _remque(caddr_t entryp) 974 {} 975 976 #else /* __lint */ 977 978 #if defined(__amd64) 979 980 ENTRY(_remque) 981 movq (%rdi), %rax /* entry->forw */ 982 movq CPTRSIZE(%rdi), %rdx /* entry->back */ 983 movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 984 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 985 ret 986 SET_SIZE(_remque) 987 988 #elif defined(__i386) 989 990 ENTRY(_remque) 991 movl 4(%esp), %ecx 992 movl (%ecx), %eax /* entry->forw */ 993 movl CPTRSIZE(%ecx), %edx /* entry->back */ 994 movl %eax, (%edx) /* entry->back->forw = entry->forw */ 995 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */ 996 ret 997 SET_SIZE(_remque) 998 999 #endif /* __i386 */ 1000 #endif /* __lint */ 1001 1002 /* 1003 * Returns the number of 1004 * non-NULL bytes in string argument. 1005 */ 1006 1007 #if defined(__lint) 1008 1009 /* ARGSUSED */ 1010 size_t 1011 strlen(const char *str) 1012 { return (0); } 1013 1014 #else /* __lint */ 1015 1016 #if defined(__amd64) 1017 1018 /* 1019 * This is close to a simple transliteration of a C version of this 1020 * routine. We should either just -make- this be a C version, or 1021 * justify having it in assembler by making it significantly faster. 1022 * 1023 * size_t 1024 * strlen(const char *s) 1025 * { 1026 * const char *s0; 1027 * #if defined(DEBUG) 1028 * if ((uintptr_t)s < KERNELBASE) 1029 * panic(.str_panic_msg); 1030 * #endif 1031 * for (s0 = s; *s; s++) 1032 * ; 1033 * return (s - s0); 1034 * } 1035 */ 1036 1037 ENTRY(strlen) 1038 #ifdef DEBUG 1039 movq postbootkernelbase(%rip), %rax 1040 cmpq %rax, %rdi 1041 jae str_valid 1042 pushq %rbp 1043 movq %rsp, %rbp 1044 leaq .str_panic_msg(%rip), %rdi 1045 xorl %eax, %eax 1046 call panic 1047 #endif /* DEBUG */ 1048 str_valid: 1049 cmpb $0, (%rdi) 1050 movq %rdi, %rax 1051 je .null_found 1052 .align 4 1053 .strlen_loop: 1054 incq %rdi 1055 cmpb $0, (%rdi) 1056 jne .strlen_loop 1057 .null_found: 1058 subq %rax, %rdi 1059 movq %rdi, %rax 1060 ret 1061 SET_SIZE(strlen) 1062 1063 #elif defined(__i386) 1064 1065 ENTRY(strlen) 1066 #ifdef DEBUG 1067 movl postbootkernelbase, %eax 1068 cmpl %eax, 4(%esp) 1069 jae str_valid 1070 pushl %ebp 1071 movl %esp, %ebp 1072 pushl $.str_panic_msg 1073 call panic 1074 #endif /* DEBUG */ 1075 1076 str_valid: 1077 movl 4(%esp), %eax /* %eax = string address */ 1078 testl $3, %eax /* if %eax not word aligned */ 1079 jnz .not_word_aligned /* goto .not_word_aligned */ 1080 .align 4 1081 .word_aligned: 1082 movl (%eax), %edx /* move 1 word from (%eax) to %edx */ 1083 movl $0x7f7f7f7f, %ecx 1084 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */ 1085 addl $4, %eax /* next word */ 1086 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */ 1087 orl %edx, %ecx /* %ecx |= %edx */ 1088 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */ 1089 cmpl $0x80808080, %ecx /* if no null byte in this word */ 1090 je .word_aligned /* goto .word_aligned */ 1091 subl $4, %eax /* post-incremented */ 1092 .not_word_aligned: 1093 cmpb $0, (%eax) /* if a byte in (%eax) is null */ 1094 je .null_found /* goto .null_found */ 1095 incl %eax /* next byte */ 1096 testl $3, %eax /* if %eax not word aligned */ 1097 jnz .not_word_aligned /* goto .not_word_aligned */ 1098 jmp .word_aligned /* goto .word_aligned */ 1099 .align 4 1100 .null_found: 1101 subl 4(%esp), %eax /* %eax -= string address */ 1102 ret 1103 SET_SIZE(strlen) 1104 1105 #endif /* __i386 */ 1106 1107 #ifdef DEBUG 1108 .text 1109 .str_panic_msg: 1110 .string "strlen: argument below kernelbase" 1111 #endif /* DEBUG */ 1112 1113 #endif /* __lint */ 1114 1115 /* 1116 * Berkeley 4.3 introduced symbolically named interrupt levels 1117 * as a way deal with priority in a machine independent fashion. 1118 * Numbered priorities are machine specific, and should be 1119 * discouraged where possible. 1120 * 1121 * Note, for the machine specific priorities there are 1122 * examples listed for devices that use a particular priority. 1123 * It should not be construed that all devices of that 1124 * type should be at that priority. It is currently were 1125 * the current devices fit into the priority scheme based 1126 * upon time criticalness. 1127 * 1128 * The underlying assumption of these assignments is that 1129 * IPL 10 is the highest level from which a device 1130 * routine can call wakeup. Devices that interrupt from higher 1131 * levels are restricted in what they can do. If they need 1132 * kernels services they should schedule a routine at a lower 1133 * level (via software interrupt) to do the required 1134 * processing. 1135 * 1136 * Examples of this higher usage: 1137 * Level Usage 1138 * 14 Profiling clock (and PROM uart polling clock) 1139 * 12 Serial ports 1140 * 1141 * The serial ports request lower level processing on level 6. 1142 * 1143 * Also, almost all splN routines (where N is a number or a 1144 * mnemonic) will do a RAISE(), on the assumption that they are 1145 * never used to lower our priority. 1146 * The exceptions are: 1147 * spl8() Because you can't be above 15 to begin with! 1148 * splzs() Because this is used at boot time to lower our 1149 * priority, to allow the PROM to poll the uart. 1150 * spl0() Used to lower priority to 0. 1151 */ 1152 1153 #if defined(__lint) 1154 1155 int spl0(void) { return (0); } 1156 int spl6(void) { return (0); } 1157 int spl7(void) { return (0); } 1158 int spl8(void) { return (0); } 1159 int splhigh(void) { return (0); } 1160 int splhi(void) { return (0); } 1161 int splzs(void) { return (0); } 1162 1163 /* ARGSUSED */ 1164 void 1165 splx(int level) 1166 {} 1167 1168 #else /* __lint */ 1169 1170 #if defined(__amd64) 1171 1172 #define SETPRI(level) \ 1173 movl $/**/level, %edi; /* new priority */ \ 1174 jmp do_splx /* redirect to do_splx */ 1175 1176 #define RAISE(level) \ 1177 movl $/**/level, %edi; /* new priority */ \ 1178 jmp splr /* redirect to splr */ 1179 1180 #elif defined(__i386) 1181 1182 #define SETPRI(level) \ 1183 pushl $/**/level; /* new priority */ \ 1184 call do_splx; /* invoke common splx code */ \ 1185 addl $4, %esp; /* unstack arg */ \ 1186 ret 1187 1188 #define RAISE(level) \ 1189 pushl $/**/level; /* new priority */ \ 1190 call splr; /* invoke common splr code */ \ 1191 addl $4, %esp; /* unstack args */ \ 1192 ret 1193 1194 #endif /* __i386 */ 1195 1196 /* locks out all interrupts, including memory errors */ 1197 ENTRY(spl8) 1198 SETPRI(15) 1199 SET_SIZE(spl8) 1200 1201 /* just below the level that profiling runs */ 1202 ENTRY(spl7) 1203 RAISE(13) 1204 SET_SIZE(spl7) 1205 1206 /* sun specific - highest priority onboard serial i/o asy ports */ 1207 ENTRY(splzs) 1208 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 1209 SET_SIZE(splzs) 1210 1211 ENTRY(splhi) 1212 ALTENTRY(splhigh) 1213 ALTENTRY(spl6) 1214 ALTENTRY(i_ddi_splhigh) 1215 1216 RAISE(DISP_LEVEL) 1217 1218 SET_SIZE(i_ddi_splhigh) 1219 SET_SIZE(spl6) 1220 SET_SIZE(splhigh) 1221 SET_SIZE(splhi) 1222 1223 /* allow all interrupts */ 1224 ENTRY(spl0) 1225 SETPRI(0) 1226 SET_SIZE(spl0) 1227 1228 1229 /* splx implementation */ 1230 ENTRY(splx) 1231 jmp do_splx /* redirect to common splx code */ 1232 SET_SIZE(splx) 1233 1234 #endif /* __lint */ 1235 1236 #if defined(__i386) 1237 1238 /* 1239 * Read and write the %gs register 1240 */ 1241 1242 #if defined(__lint) 1243 1244 /*ARGSUSED*/ 1245 uint16_t 1246 getgs(void) 1247 { return (0); } 1248 1249 /*ARGSUSED*/ 1250 void 1251 setgs(uint16_t sel) 1252 {} 1253 1254 #else /* __lint */ 1255 1256 ENTRY(getgs) 1257 clr %eax 1258 movw %gs, %ax 1259 ret 1260 SET_SIZE(getgs) 1261 1262 ENTRY(setgs) 1263 movw 4(%esp), %gs 1264 ret 1265 SET_SIZE(setgs) 1266 1267 #endif /* __lint */ 1268 #endif /* __i386 */ 1269 1270 #if defined(__lint) 1271 1272 void 1273 pc_reset(void) 1274 {} 1275 1276 void 1277 efi_reset(void) 1278 {} 1279 1280 #else /* __lint */ 1281 1282 ENTRY(wait_500ms) 1283 #if defined(__amd64) 1284 pushq %rbx 1285 #elif defined(__i386) 1286 push %ebx 1287 #endif 1288 movl $50000, %ebx 1289 1: 1290 call tenmicrosec 1291 decl %ebx 1292 jnz 1b 1293 #if defined(__amd64) 1294 popq %rbx 1295 #elif defined(__i386) 1296 pop %ebx 1297 #endif 1298 ret 1299 SET_SIZE(wait_500ms) 1300 1301 #define RESET_METHOD_KBC 1 1302 #define RESET_METHOD_PORT92 2 1303 #define RESET_METHOD_PCI 4 1304 1305 DGDEF3(pc_reset_methods, 4, 8) 1306 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 1307 1308 ENTRY(pc_reset) 1309 1310 #if defined(__i386) 1311 testl $RESET_METHOD_KBC, pc_reset_methods 1312 #elif defined(__amd64) 1313 testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 1314 #endif 1315 jz 1f 1316 1317 / 1318 / Try the classic keyboard controller-triggered reset. 1319 / 1320 movw $0x64, %dx 1321 movb $0xfe, %al 1322 outb (%dx) 1323 1324 / Wait up to 500 milliseconds here for the keyboard controller 1325 / to pull the reset line. On some systems where the keyboard 1326 / controller is slow to pull the reset line, the next reset method 1327 / may be executed (which may be bad if those systems hang when the 1328 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 1329 / and Ferrari 4000 (doesn't like the cf9 reset method)) 1330 1331 call wait_500ms 1332 1333 1: 1334 #if defined(__i386) 1335 testl $RESET_METHOD_PORT92, pc_reset_methods 1336 #elif defined(__amd64) 1337 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 1338 #endif 1339 jz 3f 1340 1341 / 1342 / Try port 0x92 fast reset 1343 / 1344 movw $0x92, %dx 1345 inb (%dx) 1346 cmpb $0xff, %al / If port's not there, we should get back 0xFF 1347 je 1f 1348 testb $1, %al / If bit 0 1349 jz 2f / is clear, jump to perform the reset 1350 andb $0xfe, %al / otherwise, 1351 outb (%dx) / clear bit 0 first, then 1352 2: 1353 orb $1, %al / Set bit 0 1354 outb (%dx) / and reset the system 1355 1: 1356 1357 call wait_500ms 1358 1359 3: 1360 #if defined(__i386) 1361 testl $RESET_METHOD_PCI, pc_reset_methods 1362 #elif defined(__amd64) 1363 testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 1364 #endif 1365 jz 4f 1366 1367 / Try the PCI (soft) reset vector (should work on all modern systems, 1368 / but has been shown to cause problems on 450NX systems, and some newer 1369 / systems (e.g. ATI IXP400-equipped systems)) 1370 / When resetting via this method, 2 writes are required. The first 1371 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 1372 / power cycle). 1373 / The reset occurs on the second write, during bit 2's transition from 1374 / 0->1. 1375 movw $0xcf9, %dx 1376 movb $0x2, %al / Reset mode = hard, no power cycle 1377 outb (%dx) 1378 movb $0x6, %al 1379 outb (%dx) 1380 1381 call wait_500ms 1382 1383 4: 1384 / 1385 / port 0xcf9 failed also. Last-ditch effort is to 1386 / triple-fault the CPU. 1387 / Also, use triple fault for EFI firmware 1388 / 1389 ENTRY(efi_reset) 1390 #if defined(__amd64) 1391 pushq $0x0 1392 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1393 lidt (%rsp) 1394 #elif defined(__i386) 1395 pushl $0x0 1396 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1397 lidt (%esp) 1398 #endif 1399 int $0x0 / Trigger interrupt, generate triple-fault 1400 1401 cli 1402 hlt / Wait forever 1403 /*NOTREACHED*/ 1404 SET_SIZE(efi_reset) 1405 SET_SIZE(pc_reset) 1406 1407 #endif /* __lint */ 1408 1409 /* 1410 * C callable in and out routines 1411 */ 1412 1413 #if defined(__lint) 1414 1415 /* ARGSUSED */ 1416 void 1417 outl(int port_address, uint32_t val) 1418 {} 1419 1420 #else /* __lint */ 1421 1422 #if defined(__amd64) 1423 1424 ENTRY(outl) 1425 movw %di, %dx 1426 movl %esi, %eax 1427 outl (%dx) 1428 ret 1429 SET_SIZE(outl) 1430 1431 #elif defined(__i386) 1432 1433 .set PORT, 4 1434 .set VAL, 8 1435 1436 ENTRY(outl) 1437 movw PORT(%esp), %dx 1438 movl VAL(%esp), %eax 1439 outl (%dx) 1440 ret 1441 SET_SIZE(outl) 1442 1443 #endif /* __i386 */ 1444 #endif /* __lint */ 1445 1446 #if defined(__lint) 1447 1448 /* ARGSUSED */ 1449 void 1450 outw(int port_address, uint16_t val) 1451 {} 1452 1453 #else /* __lint */ 1454 1455 #if defined(__amd64) 1456 1457 ENTRY(outw) 1458 movw %di, %dx 1459 movw %si, %ax 1460 D16 outl (%dx) /* XX64 why not outw? */ 1461 ret 1462 SET_SIZE(outw) 1463 1464 #elif defined(__i386) 1465 1466 ENTRY(outw) 1467 movw PORT(%esp), %dx 1468 movw VAL(%esp), %ax 1469 D16 outl (%dx) 1470 ret 1471 SET_SIZE(outw) 1472 1473 #endif /* __i386 */ 1474 #endif /* __lint */ 1475 1476 #if defined(__lint) 1477 1478 /* ARGSUSED */ 1479 void 1480 outb(int port_address, uint8_t val) 1481 {} 1482 1483 #else /* __lint */ 1484 1485 #if defined(__amd64) 1486 1487 ENTRY(outb) 1488 movw %di, %dx 1489 movb %sil, %al 1490 outb (%dx) 1491 ret 1492 SET_SIZE(outb) 1493 1494 #elif defined(__i386) 1495 1496 ENTRY(outb) 1497 movw PORT(%esp), %dx 1498 movb VAL(%esp), %al 1499 outb (%dx) 1500 ret 1501 SET_SIZE(outb) 1502 1503 #endif /* __i386 */ 1504 #endif /* __lint */ 1505 1506 #if defined(__lint) 1507 1508 /* ARGSUSED */ 1509 uint32_t 1510 inl(int port_address) 1511 { return (0); } 1512 1513 #else /* __lint */ 1514 1515 #if defined(__amd64) 1516 1517 ENTRY(inl) 1518 xorl %eax, %eax 1519 movw %di, %dx 1520 inl (%dx) 1521 ret 1522 SET_SIZE(inl) 1523 1524 #elif defined(__i386) 1525 1526 ENTRY(inl) 1527 movw PORT(%esp), %dx 1528 inl (%dx) 1529 ret 1530 SET_SIZE(inl) 1531 1532 #endif /* __i386 */ 1533 #endif /* __lint */ 1534 1535 #if defined(__lint) 1536 1537 /* ARGSUSED */ 1538 uint16_t 1539 inw(int port_address) 1540 { return (0); } 1541 1542 #else /* __lint */ 1543 1544 #if defined(__amd64) 1545 1546 ENTRY(inw) 1547 xorl %eax, %eax 1548 movw %di, %dx 1549 D16 inl (%dx) 1550 ret 1551 SET_SIZE(inw) 1552 1553 #elif defined(__i386) 1554 1555 ENTRY(inw) 1556 subl %eax, %eax 1557 movw PORT(%esp), %dx 1558 D16 inl (%dx) 1559 ret 1560 SET_SIZE(inw) 1561 1562 #endif /* __i386 */ 1563 #endif /* __lint */ 1564 1565 1566 #if defined(__lint) 1567 1568 /* ARGSUSED */ 1569 uint8_t 1570 inb(int port_address) 1571 { return (0); } 1572 1573 #else /* __lint */ 1574 1575 #if defined(__amd64) 1576 1577 ENTRY(inb) 1578 xorl %eax, %eax 1579 movw %di, %dx 1580 inb (%dx) 1581 ret 1582 SET_SIZE(inb) 1583 1584 #elif defined(__i386) 1585 1586 ENTRY(inb) 1587 subl %eax, %eax 1588 movw PORT(%esp), %dx 1589 inb (%dx) 1590 ret 1591 SET_SIZE(inb) 1592 1593 #endif /* __i386 */ 1594 #endif /* __lint */ 1595 1596 1597 #if defined(__lint) 1598 1599 /* ARGSUSED */ 1600 void 1601 repoutsw(int port, uint16_t *addr, int cnt) 1602 {} 1603 1604 #else /* __lint */ 1605 1606 #if defined(__amd64) 1607 1608 ENTRY(repoutsw) 1609 movl %edx, %ecx 1610 movw %di, %dx 1611 rep 1612 D16 outsl 1613 ret 1614 SET_SIZE(repoutsw) 1615 1616 #elif defined(__i386) 1617 1618 /* 1619 * The arguments and saved registers are on the stack in the 1620 * following order: 1621 * | cnt | +16 1622 * | *addr | +12 1623 * | port | +8 1624 * | eip | +4 1625 * | esi | <-- %esp 1626 * If additional values are pushed onto the stack, make sure 1627 * to adjust the following constants accordingly. 1628 */ 1629 .set PORT, 8 1630 .set ADDR, 12 1631 .set COUNT, 16 1632 1633 ENTRY(repoutsw) 1634 pushl %esi 1635 movl PORT(%esp), %edx 1636 movl ADDR(%esp), %esi 1637 movl COUNT(%esp), %ecx 1638 rep 1639 D16 outsl 1640 popl %esi 1641 ret 1642 SET_SIZE(repoutsw) 1643 1644 #endif /* __i386 */ 1645 #endif /* __lint */ 1646 1647 1648 #if defined(__lint) 1649 1650 /* ARGSUSED */ 1651 void 1652 repinsw(int port_addr, uint16_t *addr, int cnt) 1653 {} 1654 1655 #else /* __lint */ 1656 1657 #if defined(__amd64) 1658 1659 ENTRY(repinsw) 1660 movl %edx, %ecx 1661 movw %di, %dx 1662 rep 1663 D16 insl 1664 ret 1665 SET_SIZE(repinsw) 1666 1667 #elif defined(__i386) 1668 1669 ENTRY(repinsw) 1670 pushl %edi 1671 movl PORT(%esp), %edx 1672 movl ADDR(%esp), %edi 1673 movl COUNT(%esp), %ecx 1674 rep 1675 D16 insl 1676 popl %edi 1677 ret 1678 SET_SIZE(repinsw) 1679 1680 #endif /* __i386 */ 1681 #endif /* __lint */ 1682 1683 1684 #if defined(__lint) 1685 1686 /* ARGSUSED */ 1687 void 1688 repinsb(int port, uint8_t *addr, int count) 1689 {} 1690 1691 #else /* __lint */ 1692 1693 #if defined(__amd64) 1694 1695 ENTRY(repinsb) 1696 movl %edx, %ecx 1697 movw %di, %dx 1698 movq %rsi, %rdi 1699 rep 1700 insb 1701 ret 1702 SET_SIZE(repinsb) 1703 1704 #elif defined(__i386) 1705 1706 /* 1707 * The arguments and saved registers are on the stack in the 1708 * following order: 1709 * | cnt | +16 1710 * | *addr | +12 1711 * | port | +8 1712 * | eip | +4 1713 * | esi | <-- %esp 1714 * If additional values are pushed onto the stack, make sure 1715 * to adjust the following constants accordingly. 1716 */ 1717 .set IO_PORT, 8 1718 .set IO_ADDR, 12 1719 .set IO_COUNT, 16 1720 1721 ENTRY(repinsb) 1722 pushl %edi 1723 movl IO_ADDR(%esp), %edi 1724 movl IO_COUNT(%esp), %ecx 1725 movl IO_PORT(%esp), %edx 1726 rep 1727 insb 1728 popl %edi 1729 ret 1730 SET_SIZE(repinsb) 1731 1732 #endif /* __i386 */ 1733 #endif /* __lint */ 1734 1735 1736 /* 1737 * Input a stream of 32-bit words. 1738 * NOTE: count is a DWORD count. 1739 */ 1740 #if defined(__lint) 1741 1742 /* ARGSUSED */ 1743 void 1744 repinsd(int port, uint32_t *addr, int count) 1745 {} 1746 1747 #else /* __lint */ 1748 1749 #if defined(__amd64) 1750 1751 ENTRY(repinsd) 1752 movl %edx, %ecx 1753 movw %di, %dx 1754 movq %rsi, %rdi 1755 rep 1756 insl 1757 ret 1758 SET_SIZE(repinsd) 1759 1760 #elif defined(__i386) 1761 1762 ENTRY(repinsd) 1763 pushl %edi 1764 movl IO_ADDR(%esp), %edi 1765 movl IO_COUNT(%esp), %ecx 1766 movl IO_PORT(%esp), %edx 1767 rep 1768 insl 1769 popl %edi 1770 ret 1771 SET_SIZE(repinsd) 1772 1773 #endif /* __i386 */ 1774 #endif /* __lint */ 1775 1776 /* 1777 * Output a stream of bytes 1778 * NOTE: count is a byte count 1779 */ 1780 #if defined(__lint) 1781 1782 /* ARGSUSED */ 1783 void 1784 repoutsb(int port, uint8_t *addr, int count) 1785 {} 1786 1787 #else /* __lint */ 1788 1789 #if defined(__amd64) 1790 1791 ENTRY(repoutsb) 1792 movl %edx, %ecx 1793 movw %di, %dx 1794 rep 1795 outsb 1796 ret 1797 SET_SIZE(repoutsb) 1798 1799 #elif defined(__i386) 1800 1801 ENTRY(repoutsb) 1802 pushl %esi 1803 movl IO_ADDR(%esp), %esi 1804 movl IO_COUNT(%esp), %ecx 1805 movl IO_PORT(%esp), %edx 1806 rep 1807 outsb 1808 popl %esi 1809 ret 1810 SET_SIZE(repoutsb) 1811 1812 #endif /* __i386 */ 1813 #endif /* __lint */ 1814 1815 /* 1816 * Output a stream of 32-bit words 1817 * NOTE: count is a DWORD count 1818 */ 1819 #if defined(__lint) 1820 1821 /* ARGSUSED */ 1822 void 1823 repoutsd(int port, uint32_t *addr, int count) 1824 {} 1825 1826 #else /* __lint */ 1827 1828 #if defined(__amd64) 1829 1830 ENTRY(repoutsd) 1831 movl %edx, %ecx 1832 movw %di, %dx 1833 rep 1834 outsl 1835 ret 1836 SET_SIZE(repoutsd) 1837 1838 #elif defined(__i386) 1839 1840 ENTRY(repoutsd) 1841 pushl %esi 1842 movl IO_ADDR(%esp), %esi 1843 movl IO_COUNT(%esp), %ecx 1844 movl IO_PORT(%esp), %edx 1845 rep 1846 outsl 1847 popl %esi 1848 ret 1849 SET_SIZE(repoutsd) 1850 1851 #endif /* __i386 */ 1852 #endif /* __lint */ 1853 1854 /* 1855 * void int3(void) 1856 * void int18(void) 1857 * void int20(void) 1858 * void int_cmci(void) 1859 */ 1860 1861 #if defined(__lint) 1862 1863 void 1864 int3(void) 1865 {} 1866 1867 void 1868 int18(void) 1869 {} 1870 1871 void 1872 int20(void) 1873 {} 1874 1875 void 1876 int_cmci(void) 1877 {} 1878 1879 #else /* __lint */ 1880 1881 ENTRY(int3) 1882 int $T_BPTFLT 1883 ret 1884 SET_SIZE(int3) 1885 1886 ENTRY(int18) 1887 int $T_MCE 1888 ret 1889 SET_SIZE(int18) 1890 1891 ENTRY(int20) 1892 movl boothowto, %eax 1893 andl $RB_DEBUG, %eax 1894 jz 1f 1895 1896 int $T_DBGENTR 1897 1: 1898 rep; ret /* use 2 byte return instruction when branch target */ 1899 /* AMD Software Optimization Guide - Section 6.2 */ 1900 SET_SIZE(int20) 1901 1902 ENTRY(int_cmci) 1903 int $T_ENOEXTFLT 1904 ret 1905 SET_SIZE(int_cmci) 1906 1907 #endif /* __lint */ 1908 1909 #if defined(__lint) 1910 1911 /* ARGSUSED */ 1912 int 1913 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask) 1914 { return (0); } 1915 1916 #else /* __lint */ 1917 1918 #if defined(__amd64) 1919 1920 ENTRY(scanc) 1921 /* rdi == size */ 1922 /* rsi == cp */ 1923 /* rdx == table */ 1924 /* rcx == mask */ 1925 addq %rsi, %rdi /* end = &cp[size] */ 1926 .scanloop: 1927 cmpq %rdi, %rsi /* while (cp < end */ 1928 jnb .scandone 1929 movzbq (%rsi), %r8 /* %r8 = *cp */ 1930 incq %rsi /* cp++ */ 1931 testb %cl, (%r8, %rdx) 1932 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1933 decq %rsi /* (fix post-increment) */ 1934 .scandone: 1935 movl %edi, %eax 1936 subl %esi, %eax /* return (end - cp) */ 1937 ret 1938 SET_SIZE(scanc) 1939 1940 #elif defined(__i386) 1941 1942 ENTRY(scanc) 1943 pushl %edi 1944 pushl %esi 1945 movb 24(%esp), %cl /* mask = %cl */ 1946 movl 16(%esp), %esi /* cp = %esi */ 1947 movl 20(%esp), %edx /* table = %edx */ 1948 movl %esi, %edi 1949 addl 12(%esp), %edi /* end = &cp[size]; */ 1950 .scanloop: 1951 cmpl %edi, %esi /* while (cp < end */ 1952 jnb .scandone 1953 movzbl (%esi), %eax /* %al = *cp */ 1954 incl %esi /* cp++ */ 1955 movb (%edx, %eax), %al /* %al = table[*cp] */ 1956 testb %al, %cl 1957 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1958 dec %esi /* post-incremented */ 1959 .scandone: 1960 movl %edi, %eax 1961 subl %esi, %eax /* return (end - cp) */ 1962 popl %esi 1963 popl %edi 1964 ret 1965 SET_SIZE(scanc) 1966 1967 #endif /* __i386 */ 1968 #endif /* __lint */ 1969 1970 /* 1971 * Replacement functions for ones that are normally inlined. 1972 * In addition to the copy in i86.il, they are defined here just in case. 1973 */ 1974 1975 #if defined(__lint) 1976 1977 ulong_t 1978 intr_clear(void) 1979 { return (0); } 1980 1981 ulong_t 1982 clear_int_flag(void) 1983 { return (0); } 1984 1985 #else /* __lint */ 1986 1987 #if defined(__amd64) 1988 1989 ENTRY(intr_clear) 1990 ENTRY(clear_int_flag) 1991 pushfq 1992 popq %rax 1993 #if defined(__xpv) 1994 leaq xpv_panicking, %rdi 1995 movl (%rdi), %edi 1996 cmpl $0, %edi 1997 jne 2f 1998 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 1999 /* 2000 * Synthesize the PS_IE bit from the event mask bit 2001 */ 2002 andq $_BITNOT(PS_IE), %rax 2003 testb $1, %dl 2004 jnz 1f 2005 orq $PS_IE, %rax 2006 1: 2007 ret 2008 2: 2009 #endif 2010 CLI(%rdi) 2011 ret 2012 SET_SIZE(clear_int_flag) 2013 SET_SIZE(intr_clear) 2014 2015 #elif defined(__i386) 2016 2017 ENTRY(intr_clear) 2018 ENTRY(clear_int_flag) 2019 pushfl 2020 popl %eax 2021 #if defined(__xpv) 2022 leal xpv_panicking, %edx 2023 movl (%edx), %edx 2024 cmpl $0, %edx 2025 jne 2f 2026 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2027 /* 2028 * Synthesize the PS_IE bit from the event mask bit 2029 */ 2030 andl $_BITNOT(PS_IE), %eax 2031 testb $1, %cl 2032 jnz 1f 2033 orl $PS_IE, %eax 2034 1: 2035 ret 2036 2: 2037 #endif 2038 CLI(%edx) 2039 ret 2040 SET_SIZE(clear_int_flag) 2041 SET_SIZE(intr_clear) 2042 2043 #endif /* __i386 */ 2044 #endif /* __lint */ 2045 2046 #if defined(__lint) 2047 2048 struct cpu * 2049 curcpup(void) 2050 { return 0; } 2051 2052 #else /* __lint */ 2053 2054 #if defined(__amd64) 2055 2056 ENTRY(curcpup) 2057 movq %gs:CPU_SELF, %rax 2058 ret 2059 SET_SIZE(curcpup) 2060 2061 #elif defined(__i386) 2062 2063 ENTRY(curcpup) 2064 movl %gs:CPU_SELF, %eax 2065 ret 2066 SET_SIZE(curcpup) 2067 2068 #endif /* __i386 */ 2069 #endif /* __lint */ 2070 2071 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 2072 * These functions reverse the byte order of the input parameter and returns 2073 * the result. This is to convert the byte order from host byte order 2074 * (little endian) to network byte order (big endian), or vice versa. 2075 */ 2076 2077 #if defined(__lint) 2078 2079 uint64_t 2080 htonll(uint64_t i) 2081 { return (i); } 2082 2083 uint64_t 2084 ntohll(uint64_t i) 2085 { return (i); } 2086 2087 uint32_t 2088 htonl(uint32_t i) 2089 { return (i); } 2090 2091 uint32_t 2092 ntohl(uint32_t i) 2093 { return (i); } 2094 2095 uint16_t 2096 htons(uint16_t i) 2097 { return (i); } 2098 2099 uint16_t 2100 ntohs(uint16_t i) 2101 { return (i); } 2102 2103 #else /* __lint */ 2104 2105 #if defined(__amd64) 2106 2107 ENTRY(htonll) 2108 ALTENTRY(ntohll) 2109 movq %rdi, %rax 2110 bswapq %rax 2111 ret 2112 SET_SIZE(ntohll) 2113 SET_SIZE(htonll) 2114 2115 /* XX64 there must be shorter sequences for this */ 2116 ENTRY(htonl) 2117 ALTENTRY(ntohl) 2118 movl %edi, %eax 2119 bswap %eax 2120 ret 2121 SET_SIZE(ntohl) 2122 SET_SIZE(htonl) 2123 2124 /* XX64 there must be better sequences for this */ 2125 ENTRY(htons) 2126 ALTENTRY(ntohs) 2127 movl %edi, %eax 2128 bswap %eax 2129 shrl $16, %eax 2130 ret 2131 SET_SIZE(ntohs) 2132 SET_SIZE(htons) 2133 2134 #elif defined(__i386) 2135 2136 ENTRY(htonll) 2137 ALTENTRY(ntohll) 2138 movl 4(%esp), %edx 2139 movl 8(%esp), %eax 2140 bswap %edx 2141 bswap %eax 2142 ret 2143 SET_SIZE(ntohll) 2144 SET_SIZE(htonll) 2145 2146 ENTRY(htonl) 2147 ALTENTRY(ntohl) 2148 movl 4(%esp), %eax 2149 bswap %eax 2150 ret 2151 SET_SIZE(ntohl) 2152 SET_SIZE(htonl) 2153 2154 ENTRY(htons) 2155 ALTENTRY(ntohs) 2156 movl 4(%esp), %eax 2157 bswap %eax 2158 shrl $16, %eax 2159 ret 2160 SET_SIZE(ntohs) 2161 SET_SIZE(htons) 2162 2163 #endif /* __i386 */ 2164 #endif /* __lint */ 2165 2166 2167 #if defined(__lint) 2168 2169 /* ARGSUSED */ 2170 void 2171 intr_restore(ulong_t i) 2172 { return; } 2173 2174 /* ARGSUSED */ 2175 void 2176 restore_int_flag(ulong_t i) 2177 { return; } 2178 2179 #else /* __lint */ 2180 2181 #if defined(__amd64) 2182 2183 ENTRY(intr_restore) 2184 ENTRY(restore_int_flag) 2185 testq $PS_IE, %rdi 2186 jz 1f 2187 #if defined(__xpv) 2188 leaq xpv_panicking, %rsi 2189 movl (%rsi), %esi 2190 cmpl $0, %esi 2191 jne 1f 2192 /* 2193 * Since we're -really- running unprivileged, our attempt 2194 * to change the state of the IF bit will be ignored. 2195 * The virtual IF bit is tweaked by CLI and STI. 2196 */ 2197 IE_TO_EVENT_MASK(%rsi, %rdi) 2198 #else 2199 sti 2200 #endif 2201 1: 2202 ret 2203 SET_SIZE(restore_int_flag) 2204 SET_SIZE(intr_restore) 2205 2206 #elif defined(__i386) 2207 2208 ENTRY(intr_restore) 2209 ENTRY(restore_int_flag) 2210 testl $PS_IE, 4(%esp) 2211 jz 1f 2212 #if defined(__xpv) 2213 leal xpv_panicking, %edx 2214 movl (%edx), %edx 2215 cmpl $0, %edx 2216 jne 1f 2217 /* 2218 * Since we're -really- running unprivileged, our attempt 2219 * to change the state of the IF bit will be ignored. 2220 * The virtual IF bit is tweaked by CLI and STI. 2221 */ 2222 IE_TO_EVENT_MASK(%edx, 4(%esp)) 2223 #else 2224 sti 2225 #endif 2226 1: 2227 ret 2228 SET_SIZE(restore_int_flag) 2229 SET_SIZE(intr_restore) 2230 2231 #endif /* __i386 */ 2232 #endif /* __lint */ 2233 2234 #if defined(__lint) 2235 2236 void 2237 sti(void) 2238 {} 2239 2240 void 2241 cli(void) 2242 {} 2243 2244 #else /* __lint */ 2245 2246 ENTRY(sti) 2247 STI 2248 ret 2249 SET_SIZE(sti) 2250 2251 ENTRY(cli) 2252 #if defined(__amd64) 2253 CLI(%rax) 2254 #elif defined(__i386) 2255 CLI(%eax) 2256 #endif /* __i386 */ 2257 ret 2258 SET_SIZE(cli) 2259 2260 #endif /* __lint */ 2261 2262 #if defined(__lint) 2263 2264 dtrace_icookie_t 2265 dtrace_interrupt_disable(void) 2266 { return (0); } 2267 2268 #else /* __lint */ 2269 2270 #if defined(__amd64) 2271 2272 ENTRY(dtrace_interrupt_disable) 2273 pushfq 2274 popq %rax 2275 #if defined(__xpv) 2276 leaq xpv_panicking, %rdi 2277 movl (%rdi), %edi 2278 cmpl $0, %edi 2279 jne .dtrace_interrupt_disable_done 2280 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 2281 /* 2282 * Synthesize the PS_IE bit from the event mask bit 2283 */ 2284 andq $_BITNOT(PS_IE), %rax 2285 testb $1, %dl 2286 jnz .dtrace_interrupt_disable_done 2287 orq $PS_IE, %rax 2288 #else 2289 CLI(%rdx) 2290 #endif 2291 .dtrace_interrupt_disable_done: 2292 ret 2293 SET_SIZE(dtrace_interrupt_disable) 2294 2295 #elif defined(__i386) 2296 2297 ENTRY(dtrace_interrupt_disable) 2298 pushfl 2299 popl %eax 2300 #if defined(__xpv) 2301 leal xpv_panicking, %edx 2302 movl (%edx), %edx 2303 cmpl $0, %edx 2304 jne .dtrace_interrupt_disable_done 2305 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2306 /* 2307 * Synthesize the PS_IE bit from the event mask bit 2308 */ 2309 andl $_BITNOT(PS_IE), %eax 2310 testb $1, %cl 2311 jnz .dtrace_interrupt_disable_done 2312 orl $PS_IE, %eax 2313 #else 2314 CLI(%edx) 2315 #endif 2316 .dtrace_interrupt_disable_done: 2317 ret 2318 SET_SIZE(dtrace_interrupt_disable) 2319 2320 #endif /* __i386 */ 2321 #endif /* __lint */ 2322 2323 #if defined(__lint) 2324 2325 /*ARGSUSED*/ 2326 void 2327 dtrace_interrupt_enable(dtrace_icookie_t cookie) 2328 {} 2329 2330 #else /* __lint */ 2331 2332 #if defined(__amd64) 2333 2334 ENTRY(dtrace_interrupt_enable) 2335 pushq %rdi 2336 popfq 2337 #if defined(__xpv) 2338 leaq xpv_panicking, %rdx 2339 movl (%rdx), %edx 2340 cmpl $0, %edx 2341 jne .dtrace_interrupt_enable_done 2342 /* 2343 * Since we're -really- running unprivileged, our attempt 2344 * to change the state of the IF bit will be ignored. The 2345 * virtual IF bit is tweaked by CLI and STI. 2346 */ 2347 IE_TO_EVENT_MASK(%rdx, %rdi) 2348 #endif 2349 .dtrace_interrupt_enable_done: 2350 ret 2351 SET_SIZE(dtrace_interrupt_enable) 2352 2353 #elif defined(__i386) 2354 2355 ENTRY(dtrace_interrupt_enable) 2356 movl 4(%esp), %eax 2357 pushl %eax 2358 popfl 2359 #if defined(__xpv) 2360 leal xpv_panicking, %edx 2361 movl (%edx), %edx 2362 cmpl $0, %edx 2363 jne .dtrace_interrupt_enable_done 2364 /* 2365 * Since we're -really- running unprivileged, our attempt 2366 * to change the state of the IF bit will be ignored. The 2367 * virtual IF bit is tweaked by CLI and STI. 2368 */ 2369 IE_TO_EVENT_MASK(%edx, %eax) 2370 #endif 2371 .dtrace_interrupt_enable_done: 2372 ret 2373 SET_SIZE(dtrace_interrupt_enable) 2374 2375 #endif /* __i386 */ 2376 #endif /* __lint */ 2377 2378 2379 #if defined(lint) 2380 2381 void 2382 dtrace_membar_producer(void) 2383 {} 2384 2385 void 2386 dtrace_membar_consumer(void) 2387 {} 2388 2389 #else /* __lint */ 2390 2391 ENTRY(dtrace_membar_producer) 2392 rep; ret /* use 2 byte return instruction when branch target */ 2393 /* AMD Software Optimization Guide - Section 6.2 */ 2394 SET_SIZE(dtrace_membar_producer) 2395 2396 ENTRY(dtrace_membar_consumer) 2397 rep; ret /* use 2 byte return instruction when branch target */ 2398 /* AMD Software Optimization Guide - Section 6.2 */ 2399 SET_SIZE(dtrace_membar_consumer) 2400 2401 #endif /* __lint */ 2402 2403 #if defined(__lint) 2404 2405 kthread_id_t 2406 threadp(void) 2407 { return ((kthread_id_t)0); } 2408 2409 #else /* __lint */ 2410 2411 #if defined(__amd64) 2412 2413 ENTRY(threadp) 2414 movq %gs:CPU_THREAD, %rax 2415 ret 2416 SET_SIZE(threadp) 2417 2418 #elif defined(__i386) 2419 2420 ENTRY(threadp) 2421 movl %gs:CPU_THREAD, %eax 2422 ret 2423 SET_SIZE(threadp) 2424 2425 #endif /* __i386 */ 2426 #endif /* __lint */ 2427 2428 /* 2429 * Checksum routine for Internet Protocol Headers 2430 */ 2431 2432 #if defined(__lint) 2433 2434 /* ARGSUSED */ 2435 unsigned int 2436 ip_ocsum( 2437 ushort_t *address, /* ptr to 1st message buffer */ 2438 int halfword_count, /* length of data */ 2439 unsigned int sum) /* partial checksum */ 2440 { 2441 int i; 2442 unsigned int psum = 0; /* partial sum */ 2443 2444 for (i = 0; i < halfword_count; i++, address++) { 2445 psum += *address; 2446 } 2447 2448 while ((psum >> 16) != 0) { 2449 psum = (psum & 0xffff) + (psum >> 16); 2450 } 2451 2452 psum += sum; 2453 2454 while ((psum >> 16) != 0) { 2455 psum = (psum & 0xffff) + (psum >> 16); 2456 } 2457 2458 return (psum); 2459 } 2460 2461 #else /* __lint */ 2462 2463 #if defined(__amd64) 2464 2465 ENTRY(ip_ocsum) 2466 pushq %rbp 2467 movq %rsp, %rbp 2468 #ifdef DEBUG 2469 movq postbootkernelbase(%rip), %rax 2470 cmpq %rax, %rdi 2471 jnb 1f 2472 xorl %eax, %eax 2473 movq %rdi, %rsi 2474 leaq .ip_ocsum_panic_msg(%rip), %rdi 2475 call panic 2476 /*NOTREACHED*/ 2477 .ip_ocsum_panic_msg: 2478 .string "ip_ocsum: address 0x%p below kernelbase\n" 2479 1: 2480 #endif 2481 movl %esi, %ecx /* halfword_count */ 2482 movq %rdi, %rsi /* address */ 2483 /* partial sum in %edx */ 2484 xorl %eax, %eax 2485 testl %ecx, %ecx 2486 jz .ip_ocsum_done 2487 testq $3, %rsi 2488 jnz .ip_csum_notaligned 2489 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 2490 .next_iter: 2491 /* XX64 opportunities for prefetch? */ 2492 /* XX64 compute csum with 64 bit quantities? */ 2493 subl $32, %ecx 2494 jl .less_than_32 2495 2496 addl 0(%rsi), %edx 2497 .only60: 2498 adcl 4(%rsi), %eax 2499 .only56: 2500 adcl 8(%rsi), %edx 2501 .only52: 2502 adcl 12(%rsi), %eax 2503 .only48: 2504 adcl 16(%rsi), %edx 2505 .only44: 2506 adcl 20(%rsi), %eax 2507 .only40: 2508 adcl 24(%rsi), %edx 2509 .only36: 2510 adcl 28(%rsi), %eax 2511 .only32: 2512 adcl 32(%rsi), %edx 2513 .only28: 2514 adcl 36(%rsi), %eax 2515 .only24: 2516 adcl 40(%rsi), %edx 2517 .only20: 2518 adcl 44(%rsi), %eax 2519 .only16: 2520 adcl 48(%rsi), %edx 2521 .only12: 2522 adcl 52(%rsi), %eax 2523 .only8: 2524 adcl 56(%rsi), %edx 2525 .only4: 2526 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 2527 .only0: 2528 adcl $0, %eax /* could be adding -1 in eax with a carry */ 2529 adcl $0, %eax 2530 2531 addq $64, %rsi 2532 testl %ecx, %ecx 2533 jnz .next_iter 2534 2535 .ip_ocsum_done: 2536 addl %eax, %edx 2537 adcl $0, %edx 2538 movl %edx, %eax /* form a 16 bit checksum by */ 2539 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2540 addw %dx, %ax 2541 adcw $0, %ax 2542 andl $0xffff, %eax 2543 leave 2544 ret 2545 2546 .ip_csum_notaligned: 2547 xorl %edi, %edi 2548 movw (%rsi), %di 2549 addl %edi, %edx 2550 adcl $0, %edx 2551 addq $2, %rsi 2552 decl %ecx 2553 jmp .ip_csum_aligned 2554 2555 .less_than_32: 2556 addl $32, %ecx 2557 testl $1, %ecx 2558 jz .size_aligned 2559 andl $0xfe, %ecx 2560 movzwl (%rsi, %rcx, 2), %edi 2561 addl %edi, %edx 2562 adcl $0, %edx 2563 .size_aligned: 2564 movl %ecx, %edi 2565 shrl $1, %ecx 2566 shl $1, %edi 2567 subq $64, %rdi 2568 addq %rdi, %rsi 2569 leaq .ip_ocsum_jmptbl(%rip), %rdi 2570 leaq (%rdi, %rcx, 8), %rdi 2571 xorl %ecx, %ecx 2572 clc 2573 jmp *(%rdi) 2574 2575 .align 8 2576 .ip_ocsum_jmptbl: 2577 .quad .only0, .only4, .only8, .only12, .only16, .only20 2578 .quad .only24, .only28, .only32, .only36, .only40, .only44 2579 .quad .only48, .only52, .only56, .only60 2580 SET_SIZE(ip_ocsum) 2581 2582 #elif defined(__i386) 2583 2584 ENTRY(ip_ocsum) 2585 pushl %ebp 2586 movl %esp, %ebp 2587 pushl %ebx 2588 pushl %esi 2589 pushl %edi 2590 movl 12(%ebp), %ecx /* count of half words */ 2591 movl 16(%ebp), %edx /* partial checksum */ 2592 movl 8(%ebp), %esi 2593 xorl %eax, %eax 2594 testl %ecx, %ecx 2595 jz .ip_ocsum_done 2596 2597 testl $3, %esi 2598 jnz .ip_csum_notaligned 2599 .ip_csum_aligned: 2600 .next_iter: 2601 subl $32, %ecx 2602 jl .less_than_32 2603 2604 addl 0(%esi), %edx 2605 .only60: 2606 adcl 4(%esi), %eax 2607 .only56: 2608 adcl 8(%esi), %edx 2609 .only52: 2610 adcl 12(%esi), %eax 2611 .only48: 2612 adcl 16(%esi), %edx 2613 .only44: 2614 adcl 20(%esi), %eax 2615 .only40: 2616 adcl 24(%esi), %edx 2617 .only36: 2618 adcl 28(%esi), %eax 2619 .only32: 2620 adcl 32(%esi), %edx 2621 .only28: 2622 adcl 36(%esi), %eax 2623 .only24: 2624 adcl 40(%esi), %edx 2625 .only20: 2626 adcl 44(%esi), %eax 2627 .only16: 2628 adcl 48(%esi), %edx 2629 .only12: 2630 adcl 52(%esi), %eax 2631 .only8: 2632 adcl 56(%esi), %edx 2633 .only4: 2634 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */ 2635 .only0: 2636 adcl $0, %eax /* we could be adding -1 in eax with a carry */ 2637 adcl $0, %eax 2638 2639 addl $64, %esi 2640 andl %ecx, %ecx 2641 jnz .next_iter 2642 2643 .ip_ocsum_done: 2644 addl %eax, %edx 2645 adcl $0, %edx 2646 movl %edx, %eax /* form a 16 bit checksum by */ 2647 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2648 addw %dx, %ax 2649 adcw $0, %ax 2650 andl $0xffff, %eax 2651 popl %edi /* restore registers */ 2652 popl %esi 2653 popl %ebx 2654 leave 2655 ret 2656 2657 .ip_csum_notaligned: 2658 xorl %edi, %edi 2659 movw (%esi), %di 2660 addl %edi, %edx 2661 adcl $0, %edx 2662 addl $2, %esi 2663 decl %ecx 2664 jmp .ip_csum_aligned 2665 2666 .less_than_32: 2667 addl $32, %ecx 2668 testl $1, %ecx 2669 jz .size_aligned 2670 andl $0xfe, %ecx 2671 movzwl (%esi, %ecx, 2), %edi 2672 addl %edi, %edx 2673 adcl $0, %edx 2674 .size_aligned: 2675 movl %ecx, %edi 2676 shrl $1, %ecx 2677 shl $1, %edi 2678 subl $64, %edi 2679 addl %edi, %esi 2680 movl $.ip_ocsum_jmptbl, %edi 2681 lea (%edi, %ecx, 4), %edi 2682 xorl %ecx, %ecx 2683 clc 2684 jmp *(%edi) 2685 SET_SIZE(ip_ocsum) 2686 2687 .data 2688 .align 4 2689 2690 .ip_ocsum_jmptbl: 2691 .long .only0, .only4, .only8, .only12, .only16, .only20 2692 .long .only24, .only28, .only32, .only36, .only40, .only44 2693 .long .only48, .only52, .only56, .only60 2694 2695 2696 #endif /* __i386 */ 2697 #endif /* __lint */ 2698 2699 /* 2700 * multiply two long numbers and yield a u_longlong_t result, callable from C. 2701 * Provided to manipulate hrtime_t values. 2702 */ 2703 #if defined(__lint) 2704 2705 /* result = a * b; */ 2706 2707 /* ARGSUSED */ 2708 unsigned long long 2709 mul32(uint_t a, uint_t b) 2710 { return (0); } 2711 2712 #else /* __lint */ 2713 2714 #if defined(__amd64) 2715 2716 ENTRY(mul32) 2717 xorl %edx, %edx /* XX64 joe, paranoia? */ 2718 movl %edi, %eax 2719 mull %esi 2720 shlq $32, %rdx 2721 orq %rdx, %rax 2722 ret 2723 SET_SIZE(mul32) 2724 2725 #elif defined(__i386) 2726 2727 ENTRY(mul32) 2728 movl 8(%esp), %eax 2729 movl 4(%esp), %ecx 2730 mull %ecx 2731 ret 2732 SET_SIZE(mul32) 2733 2734 #endif /* __i386 */ 2735 #endif /* __lint */ 2736 2737 #if defined(notused) 2738 #if defined(__lint) 2739 /* ARGSUSED */ 2740 void 2741 load_pte64(uint64_t *pte, uint64_t pte_value) 2742 {} 2743 #else /* __lint */ 2744 .globl load_pte64 2745 load_pte64: 2746 movl 4(%esp), %eax 2747 movl 8(%esp), %ecx 2748 movl 12(%esp), %edx 2749 movl %edx, 4(%eax) 2750 movl %ecx, (%eax) 2751 ret 2752 #endif /* __lint */ 2753 #endif /* notused */ 2754 2755 #if defined(__lint) 2756 2757 /*ARGSUSED*/ 2758 void 2759 scan_memory(caddr_t addr, size_t size) 2760 {} 2761 2762 #else /* __lint */ 2763 2764 #if defined(__amd64) 2765 2766 ENTRY(scan_memory) 2767 shrq $3, %rsi /* convert %rsi from byte to quadword count */ 2768 jz .scanm_done 2769 movq %rsi, %rcx /* move count into rep control register */ 2770 movq %rdi, %rsi /* move addr into lodsq control reg. */ 2771 rep lodsq /* scan the memory range */ 2772 .scanm_done: 2773 rep; ret /* use 2 byte return instruction when branch target */ 2774 /* AMD Software Optimization Guide - Section 6.2 */ 2775 SET_SIZE(scan_memory) 2776 2777 #elif defined(__i386) 2778 2779 ENTRY(scan_memory) 2780 pushl %ecx 2781 pushl %esi 2782 movl 16(%esp), %ecx /* move 2nd arg into rep control register */ 2783 shrl $2, %ecx /* convert from byte count to word count */ 2784 jz .scanm_done 2785 movl 12(%esp), %esi /* move 1st arg into lodsw control register */ 2786 .byte 0xf3 /* rep prefix. lame assembler. sigh. */ 2787 lodsl 2788 .scanm_done: 2789 popl %esi 2790 popl %ecx 2791 ret 2792 SET_SIZE(scan_memory) 2793 2794 #endif /* __i386 */ 2795 #endif /* __lint */ 2796 2797 2798 #if defined(__lint) 2799 2800 /*ARGSUSED */ 2801 int 2802 lowbit(ulong_t i) 2803 { return (0); } 2804 2805 #else /* __lint */ 2806 2807 #if defined(__amd64) 2808 2809 ENTRY(lowbit) 2810 movl $-1, %eax 2811 bsfq %rdi, %rdi 2812 cmovnz %edi, %eax 2813 incl %eax 2814 ret 2815 SET_SIZE(lowbit) 2816 2817 #elif defined(__i386) 2818 2819 ENTRY(lowbit) 2820 bsfl 4(%esp), %eax 2821 jz 0f 2822 incl %eax 2823 ret 2824 0: 2825 xorl %eax, %eax 2826 ret 2827 SET_SIZE(lowbit) 2828 2829 #endif /* __i386 */ 2830 #endif /* __lint */ 2831 2832 #if defined(__lint) 2833 2834 /*ARGSUSED*/ 2835 int 2836 highbit(ulong_t i) 2837 { return (0); } 2838 2839 /*ARGSUSED*/ 2840 int 2841 highbit64(uint64_t i) 2842 { return (0); } 2843 2844 #else /* __lint */ 2845 2846 #if defined(__amd64) 2847 2848 ENTRY(highbit) 2849 ALTENTRY(highbit64) 2850 movl $-1, %eax 2851 bsrq %rdi, %rdi 2852 cmovnz %edi, %eax 2853 incl %eax 2854 ret 2855 SET_SIZE(highbit64) 2856 SET_SIZE(highbit) 2857 2858 #elif defined(__i386) 2859 2860 ENTRY(highbit) 2861 bsrl 4(%esp), %eax 2862 jz 0f 2863 incl %eax 2864 ret 2865 0: 2866 xorl %eax, %eax 2867 ret 2868 SET_SIZE(highbit) 2869 2870 ENTRY(highbit64) 2871 bsrl 8(%esp), %eax 2872 jz highbit 2873 addl $33, %eax 2874 ret 2875 SET_SIZE(highbit64) 2876 2877 #endif /* __i386 */ 2878 #endif /* __lint */ 2879 2880 #if defined(__lint) 2881 2882 /*ARGSUSED*/ 2883 uint64_t 2884 rdmsr(uint_t r) 2885 { return (0); } 2886 2887 /*ARGSUSED*/ 2888 void 2889 wrmsr(uint_t r, const uint64_t val) 2890 {} 2891 2892 /*ARGSUSED*/ 2893 uint64_t 2894 xrdmsr(uint_t r) 2895 { return (0); } 2896 2897 /*ARGSUSED*/ 2898 void 2899 xwrmsr(uint_t r, const uint64_t val) 2900 {} 2901 2902 void 2903 invalidate_cache(void) 2904 {} 2905 2906 /*ARGSUSED*/ 2907 uint64_t 2908 get_xcr(uint_t r) 2909 { return (0); } 2910 2911 /*ARGSUSED*/ 2912 void 2913 set_xcr(uint_t r, const uint64_t val) 2914 {} 2915 2916 #else /* __lint */ 2917 2918 #define XMSR_ACCESS_VAL $0x9c5a203a 2919 2920 #if defined(__amd64) 2921 2922 ENTRY(rdmsr) 2923 movl %edi, %ecx 2924 rdmsr 2925 shlq $32, %rdx 2926 orq %rdx, %rax 2927 ret 2928 SET_SIZE(rdmsr) 2929 2930 ENTRY(wrmsr) 2931 movq %rsi, %rdx 2932 shrq $32, %rdx 2933 movl %esi, %eax 2934 movl %edi, %ecx 2935 wrmsr 2936 ret 2937 SET_SIZE(wrmsr) 2938 2939 ENTRY(xrdmsr) 2940 pushq %rbp 2941 movq %rsp, %rbp 2942 movl %edi, %ecx 2943 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2944 rdmsr 2945 shlq $32, %rdx 2946 orq %rdx, %rax 2947 leave 2948 ret 2949 SET_SIZE(xrdmsr) 2950 2951 ENTRY(xwrmsr) 2952 pushq %rbp 2953 movq %rsp, %rbp 2954 movl %edi, %ecx 2955 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2956 movq %rsi, %rdx 2957 shrq $32, %rdx 2958 movl %esi, %eax 2959 wrmsr 2960 leave 2961 ret 2962 SET_SIZE(xwrmsr) 2963 2964 ENTRY(get_xcr) 2965 movl %edi, %ecx 2966 #xgetbv 2967 .byte 0x0f,0x01,0xd0 2968 shlq $32, %rdx 2969 orq %rdx, %rax 2970 ret 2971 SET_SIZE(get_xcr) 2972 2973 ENTRY(set_xcr) 2974 movq %rsi, %rdx 2975 shrq $32, %rdx 2976 movl %esi, %eax 2977 movl %edi, %ecx 2978 #xsetbv 2979 .byte 0x0f,0x01,0xd1 2980 ret 2981 SET_SIZE(set_xcr) 2982 2983 #elif defined(__i386) 2984 2985 ENTRY(rdmsr) 2986 movl 4(%esp), %ecx 2987 rdmsr 2988 ret 2989 SET_SIZE(rdmsr) 2990 2991 ENTRY(wrmsr) 2992 movl 4(%esp), %ecx 2993 movl 8(%esp), %eax 2994 movl 12(%esp), %edx 2995 wrmsr 2996 ret 2997 SET_SIZE(wrmsr) 2998 2999 ENTRY(xrdmsr) 3000 pushl %ebp 3001 movl %esp, %ebp 3002 movl 8(%esp), %ecx 3003 pushl %edi 3004 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 3005 rdmsr 3006 popl %edi 3007 leave 3008 ret 3009 SET_SIZE(xrdmsr) 3010 3011 ENTRY(xwrmsr) 3012 pushl %ebp 3013 movl %esp, %ebp 3014 movl 8(%esp), %ecx 3015 movl 12(%esp), %eax 3016 movl 16(%esp), %edx 3017 pushl %edi 3018 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 3019 wrmsr 3020 popl %edi 3021 leave 3022 ret 3023 SET_SIZE(xwrmsr) 3024 3025 ENTRY(get_xcr) 3026 movl 4(%esp), %ecx 3027 #xgetbv 3028 .byte 0x0f,0x01,0xd0 3029 ret 3030 SET_SIZE(get_xcr) 3031 3032 ENTRY(set_xcr) 3033 movl 4(%esp), %ecx 3034 movl 8(%esp), %eax 3035 movl 12(%esp), %edx 3036 #xsetbv 3037 .byte 0x0f,0x01,0xd1 3038 ret 3039 SET_SIZE(set_xcr) 3040 3041 #endif /* __i386 */ 3042 3043 ENTRY(invalidate_cache) 3044 wbinvd 3045 ret 3046 SET_SIZE(invalidate_cache) 3047 3048 #endif /* __lint */ 3049 3050 #if defined(__lint) 3051 3052 /*ARGSUSED*/ 3053 void 3054 getcregs(struct cregs *crp) 3055 {} 3056 3057 #else /* __lint */ 3058 3059 #if defined(__amd64) 3060 3061 ENTRY_NP(getcregs) 3062 #if defined(__xpv) 3063 /* 3064 * Only a few of the hardware control registers or descriptor tables 3065 * are directly accessible to us, so just zero the structure. 3066 * 3067 * XXPV Perhaps it would be helpful for the hypervisor to return 3068 * virtualized versions of these for post-mortem use. 3069 * (Need to reevaluate - perhaps it already does!) 3070 */ 3071 pushq %rdi /* save *crp */ 3072 movq $CREGSZ, %rsi 3073 call bzero 3074 popq %rdi 3075 3076 /* 3077 * Dump what limited information we can 3078 */ 3079 movq %cr0, %rax 3080 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3081 movq %cr2, %rax 3082 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3083 movq %cr3, %rax 3084 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3085 movq %cr4, %rax 3086 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3087 3088 #else /* __xpv */ 3089 3090 #define GETMSR(r, off, d) \ 3091 movl $r, %ecx; \ 3092 rdmsr; \ 3093 movl %eax, off(d); \ 3094 movl %edx, off+4(d) 3095 3096 xorl %eax, %eax 3097 movq %rax, CREG_GDT+8(%rdi) 3098 sgdt CREG_GDT(%rdi) /* 10 bytes */ 3099 movq %rax, CREG_IDT+8(%rdi) 3100 sidt CREG_IDT(%rdi) /* 10 bytes */ 3101 movq %rax, CREG_LDT(%rdi) 3102 sldt CREG_LDT(%rdi) /* 2 bytes */ 3103 movq %rax, CREG_TASKR(%rdi) 3104 str CREG_TASKR(%rdi) /* 2 bytes */ 3105 movq %cr0, %rax 3106 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3107 movq %cr2, %rax 3108 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3109 movq %cr3, %rax 3110 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3111 movq %cr4, %rax 3112 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3113 movq %cr8, %rax 3114 movq %rax, CREG_CR8(%rdi) /* cr8 */ 3115 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 3116 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 3117 #endif /* __xpv */ 3118 ret 3119 SET_SIZE(getcregs) 3120 3121 #undef GETMSR 3122 3123 #elif defined(__i386) 3124 3125 ENTRY_NP(getcregs) 3126 #if defined(__xpv) 3127 /* 3128 * Only a few of the hardware control registers or descriptor tables 3129 * are directly accessible to us, so just zero the structure. 3130 * 3131 * XXPV Perhaps it would be helpful for the hypervisor to return 3132 * virtualized versions of these for post-mortem use. 3133 * (Need to reevaluate - perhaps it already does!) 3134 */ 3135 movl 4(%esp), %edx 3136 pushl $CREGSZ 3137 pushl %edx 3138 call bzero 3139 addl $8, %esp 3140 movl 4(%esp), %edx 3141 3142 /* 3143 * Dump what limited information we can 3144 */ 3145 movl %cr0, %eax 3146 movl %eax, CREG_CR0(%edx) /* cr0 */ 3147 movl %cr2, %eax 3148 movl %eax, CREG_CR2(%edx) /* cr2 */ 3149 movl %cr3, %eax 3150 movl %eax, CREG_CR3(%edx) /* cr3 */ 3151 movl %cr4, %eax 3152 movl %eax, CREG_CR4(%edx) /* cr4 */ 3153 3154 #else /* __xpv */ 3155 3156 movl 4(%esp), %edx 3157 movw $0, CREG_GDT+6(%edx) 3158 movw $0, CREG_IDT+6(%edx) 3159 sgdt CREG_GDT(%edx) /* gdt */ 3160 sidt CREG_IDT(%edx) /* idt */ 3161 sldt CREG_LDT(%edx) /* ldt */ 3162 str CREG_TASKR(%edx) /* task */ 3163 movl %cr0, %eax 3164 movl %eax, CREG_CR0(%edx) /* cr0 */ 3165 movl %cr2, %eax 3166 movl %eax, CREG_CR2(%edx) /* cr2 */ 3167 movl %cr3, %eax 3168 movl %eax, CREG_CR3(%edx) /* cr3 */ 3169 bt $X86FSET_LARGEPAGE, x86_featureset 3170 jnc .nocr4 3171 movl %cr4, %eax 3172 movl %eax, CREG_CR4(%edx) /* cr4 */ 3173 jmp .skip 3174 .nocr4: 3175 movl $0, CREG_CR4(%edx) 3176 .skip: 3177 #endif 3178 ret 3179 SET_SIZE(getcregs) 3180 3181 #endif /* __i386 */ 3182 #endif /* __lint */ 3183 3184 3185 /* 3186 * A panic trigger is a word which is updated atomically and can only be set 3187 * once. We atomically store 0xDEFACEDD and load the old value. If the 3188 * previous value was 0, we succeed and return 1; otherwise return 0. 3189 * This allows a partially corrupt trigger to still trigger correctly. DTrace 3190 * has its own version of this function to allow it to panic correctly from 3191 * probe context. 3192 */ 3193 #if defined(__lint) 3194 3195 /*ARGSUSED*/ 3196 int 3197 panic_trigger(int *tp) 3198 { return (0); } 3199 3200 /*ARGSUSED*/ 3201 int 3202 dtrace_panic_trigger(int *tp) 3203 { return (0); } 3204 3205 #else /* __lint */ 3206 3207 #if defined(__amd64) 3208 3209 ENTRY_NP(panic_trigger) 3210 xorl %eax, %eax 3211 movl $0xdefacedd, %edx 3212 lock 3213 xchgl %edx, (%rdi) 3214 cmpl $0, %edx 3215 je 0f 3216 movl $0, %eax 3217 ret 3218 0: movl $1, %eax 3219 ret 3220 SET_SIZE(panic_trigger) 3221 3222 ENTRY_NP(dtrace_panic_trigger) 3223 xorl %eax, %eax 3224 movl $0xdefacedd, %edx 3225 lock 3226 xchgl %edx, (%rdi) 3227 cmpl $0, %edx 3228 je 0f 3229 movl $0, %eax 3230 ret 3231 0: movl $1, %eax 3232 ret 3233 SET_SIZE(dtrace_panic_trigger) 3234 3235 #elif defined(__i386) 3236 3237 ENTRY_NP(panic_trigger) 3238 movl 4(%esp), %edx / %edx = address of trigger 3239 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3240 lock / assert lock 3241 xchgl %eax, (%edx) / exchange %eax and the trigger 3242 cmpl $0, %eax / if (%eax == 0x0) 3243 je 0f / return (1); 3244 movl $0, %eax / else 3245 ret / return (0); 3246 0: movl $1, %eax 3247 ret 3248 SET_SIZE(panic_trigger) 3249 3250 ENTRY_NP(dtrace_panic_trigger) 3251 movl 4(%esp), %edx / %edx = address of trigger 3252 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3253 lock / assert lock 3254 xchgl %eax, (%edx) / exchange %eax and the trigger 3255 cmpl $0, %eax / if (%eax == 0x0) 3256 je 0f / return (1); 3257 movl $0, %eax / else 3258 ret / return (0); 3259 0: movl $1, %eax 3260 ret 3261 SET_SIZE(dtrace_panic_trigger) 3262 3263 #endif /* __i386 */ 3264 #endif /* __lint */ 3265 3266 /* 3267 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 3268 * into the panic code implemented in panicsys(). vpanic() is responsible 3269 * for passing through the format string and arguments, and constructing a 3270 * regs structure on the stack into which it saves the current register 3271 * values. If we are not dying due to a fatal trap, these registers will 3272 * then be preserved in panicbuf as the current processor state. Before 3273 * invoking panicsys(), vpanic() activates the first panic trigger (see 3274 * common/os/panic.c) and switches to the panic_stack if successful. Note that 3275 * DTrace takes a slightly different panic path if it must panic from probe 3276 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 3277 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 3278 * branches back into vpanic(). 3279 */ 3280 #if defined(__lint) 3281 3282 /*ARGSUSED*/ 3283 void 3284 vpanic(const char *format, va_list alist) 3285 {} 3286 3287 /*ARGSUSED*/ 3288 void 3289 dtrace_vpanic(const char *format, va_list alist) 3290 {} 3291 3292 #else /* __lint */ 3293 3294 #if defined(__amd64) 3295 3296 ENTRY_NP(vpanic) /* Initial stack layout: */ 3297 3298 pushq %rbp /* | %rip | 0x60 */ 3299 movq %rsp, %rbp /* | %rbp | 0x58 */ 3300 pushfq /* | rfl | 0x50 */ 3301 pushq %r11 /* | %r11 | 0x48 */ 3302 pushq %r10 /* | %r10 | 0x40 */ 3303 pushq %rbx /* | %rbx | 0x38 */ 3304 pushq %rax /* | %rax | 0x30 */ 3305 pushq %r9 /* | %r9 | 0x28 */ 3306 pushq %r8 /* | %r8 | 0x20 */ 3307 pushq %rcx /* | %rcx | 0x18 */ 3308 pushq %rdx /* | %rdx | 0x10 */ 3309 pushq %rsi /* | %rsi | 0x8 alist */ 3310 pushq %rdi /* | %rdi | 0x0 format */ 3311 3312 movq %rsp, %rbx /* %rbx = current %rsp */ 3313 3314 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3315 call panic_trigger /* %eax = panic_trigger() */ 3316 3317 vpanic_common: 3318 /* 3319 * The panic_trigger result is in %eax from the call above, and 3320 * dtrace_panic places it in %eax before branching here. 3321 * The rdmsr instructions that follow below will clobber %eax so 3322 * we stash the panic_trigger result in %r11d. 3323 */ 3324 movl %eax, %r11d 3325 cmpl $0, %r11d 3326 je 0f 3327 3328 /* 3329 * If panic_trigger() was successful, we are the first to initiate a 3330 * panic: we now switch to the reserved panic_stack before continuing. 3331 */ 3332 leaq panic_stack(%rip), %rsp 3333 addq $PANICSTKSIZE, %rsp 3334 0: subq $REGSIZE, %rsp 3335 /* 3336 * Now that we've got everything set up, store the register values as 3337 * they were when we entered vpanic() to the designated location in 3338 * the regs structure we allocated on the stack. 3339 */ 3340 movq 0x0(%rbx), %rcx 3341 movq %rcx, REGOFF_RDI(%rsp) 3342 movq 0x8(%rbx), %rcx 3343 movq %rcx, REGOFF_RSI(%rsp) 3344 movq 0x10(%rbx), %rcx 3345 movq %rcx, REGOFF_RDX(%rsp) 3346 movq 0x18(%rbx), %rcx 3347 movq %rcx, REGOFF_RCX(%rsp) 3348 movq 0x20(%rbx), %rcx 3349 3350 movq %rcx, REGOFF_R8(%rsp) 3351 movq 0x28(%rbx), %rcx 3352 movq %rcx, REGOFF_R9(%rsp) 3353 movq 0x30(%rbx), %rcx 3354 movq %rcx, REGOFF_RAX(%rsp) 3355 movq 0x38(%rbx), %rcx 3356 movq %rcx, REGOFF_RBX(%rsp) 3357 movq 0x58(%rbx), %rcx 3358 3359 movq %rcx, REGOFF_RBP(%rsp) 3360 movq 0x40(%rbx), %rcx 3361 movq %rcx, REGOFF_R10(%rsp) 3362 movq 0x48(%rbx), %rcx 3363 movq %rcx, REGOFF_R11(%rsp) 3364 movq %r12, REGOFF_R12(%rsp) 3365 3366 movq %r13, REGOFF_R13(%rsp) 3367 movq %r14, REGOFF_R14(%rsp) 3368 movq %r15, REGOFF_R15(%rsp) 3369 3370 xorl %ecx, %ecx 3371 movw %ds, %cx 3372 movq %rcx, REGOFF_DS(%rsp) 3373 movw %es, %cx 3374 movq %rcx, REGOFF_ES(%rsp) 3375 movw %fs, %cx 3376 movq %rcx, REGOFF_FS(%rsp) 3377 movw %gs, %cx 3378 movq %rcx, REGOFF_GS(%rsp) 3379 3380 movq $0, REGOFF_TRAPNO(%rsp) 3381 3382 movq $0, REGOFF_ERR(%rsp) 3383 leaq vpanic(%rip), %rcx 3384 movq %rcx, REGOFF_RIP(%rsp) 3385 movw %cs, %cx 3386 movzwq %cx, %rcx 3387 movq %rcx, REGOFF_CS(%rsp) 3388 movq 0x50(%rbx), %rcx 3389 movq %rcx, REGOFF_RFL(%rsp) 3390 movq %rbx, %rcx 3391 addq $0x60, %rcx 3392 movq %rcx, REGOFF_RSP(%rsp) 3393 movw %ss, %cx 3394 movzwq %cx, %rcx 3395 movq %rcx, REGOFF_SS(%rsp) 3396 3397 /* 3398 * panicsys(format, alist, rp, on_panic_stack) 3399 */ 3400 movq REGOFF_RDI(%rsp), %rdi /* format */ 3401 movq REGOFF_RSI(%rsp), %rsi /* alist */ 3402 movq %rsp, %rdx /* struct regs */ 3403 movl %r11d, %ecx /* on_panic_stack */ 3404 call panicsys 3405 addq $REGSIZE, %rsp 3406 popq %rdi 3407 popq %rsi 3408 popq %rdx 3409 popq %rcx 3410 popq %r8 3411 popq %r9 3412 popq %rax 3413 popq %rbx 3414 popq %r10 3415 popq %r11 3416 popfq 3417 leave 3418 ret 3419 SET_SIZE(vpanic) 3420 3421 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 3422 3423 pushq %rbp /* | %rip | 0x60 */ 3424 movq %rsp, %rbp /* | %rbp | 0x58 */ 3425 pushfq /* | rfl | 0x50 */ 3426 pushq %r11 /* | %r11 | 0x48 */ 3427 pushq %r10 /* | %r10 | 0x40 */ 3428 pushq %rbx /* | %rbx | 0x38 */ 3429 pushq %rax /* | %rax | 0x30 */ 3430 pushq %r9 /* | %r9 | 0x28 */ 3431 pushq %r8 /* | %r8 | 0x20 */ 3432 pushq %rcx /* | %rcx | 0x18 */ 3433 pushq %rdx /* | %rdx | 0x10 */ 3434 pushq %rsi /* | %rsi | 0x8 alist */ 3435 pushq %rdi /* | %rdi | 0x0 format */ 3436 3437 movq %rsp, %rbx /* %rbx = current %rsp */ 3438 3439 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3440 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 3441 jmp vpanic_common 3442 3443 SET_SIZE(dtrace_vpanic) 3444 3445 #elif defined(__i386) 3446 3447 ENTRY_NP(vpanic) / Initial stack layout: 3448 3449 pushl %ebp / | %eip | 20 3450 movl %esp, %ebp / | %ebp | 16 3451 pushl %eax / | %eax | 12 3452 pushl %ebx / | %ebx | 8 3453 pushl %ecx / | %ecx | 4 3454 pushl %edx / | %edx | 0 3455 3456 movl %esp, %ebx / %ebx = current stack pointer 3457 3458 lea panic_quiesce, %eax / %eax = &panic_quiesce 3459 pushl %eax / push &panic_quiesce 3460 call panic_trigger / %eax = panic_trigger() 3461 addl $4, %esp / reset stack pointer 3462 3463 vpanic_common: 3464 cmpl $0, %eax / if (%eax == 0) 3465 je 0f / goto 0f; 3466 3467 /* 3468 * If panic_trigger() was successful, we are the first to initiate a 3469 * panic: we now switch to the reserved panic_stack before continuing. 3470 */ 3471 lea panic_stack, %esp / %esp = panic_stack 3472 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE 3473 3474 0: subl $REGSIZE, %esp / allocate struct regs 3475 3476 /* 3477 * Now that we've got everything set up, store the register values as 3478 * they were when we entered vpanic() to the designated location in 3479 * the regs structure we allocated on the stack. 3480 */ 3481 #if !defined(__GNUC_AS__) 3482 movw %gs, %edx 3483 movl %edx, REGOFF_GS(%esp) 3484 movw %fs, %edx 3485 movl %edx, REGOFF_FS(%esp) 3486 movw %es, %edx 3487 movl %edx, REGOFF_ES(%esp) 3488 movw %ds, %edx 3489 movl %edx, REGOFF_DS(%esp) 3490 #else /* __GNUC_AS__ */ 3491 mov %gs, %edx 3492 mov %edx, REGOFF_GS(%esp) 3493 mov %fs, %edx 3494 mov %edx, REGOFF_FS(%esp) 3495 mov %es, %edx 3496 mov %edx, REGOFF_ES(%esp) 3497 mov %ds, %edx 3498 mov %edx, REGOFF_DS(%esp) 3499 #endif /* __GNUC_AS__ */ 3500 movl %edi, REGOFF_EDI(%esp) 3501 movl %esi, REGOFF_ESI(%esp) 3502 movl 16(%ebx), %ecx 3503 movl %ecx, REGOFF_EBP(%esp) 3504 movl %ebx, %ecx 3505 addl $20, %ecx 3506 movl %ecx, REGOFF_ESP(%esp) 3507 movl 8(%ebx), %ecx 3508 movl %ecx, REGOFF_EBX(%esp) 3509 movl 0(%ebx), %ecx 3510 movl %ecx, REGOFF_EDX(%esp) 3511 movl 4(%ebx), %ecx 3512 movl %ecx, REGOFF_ECX(%esp) 3513 movl 12(%ebx), %ecx 3514 movl %ecx, REGOFF_EAX(%esp) 3515 movl $0, REGOFF_TRAPNO(%esp) 3516 movl $0, REGOFF_ERR(%esp) 3517 lea vpanic, %ecx 3518 movl %ecx, REGOFF_EIP(%esp) 3519 #if !defined(__GNUC_AS__) 3520 movw %cs, %edx 3521 #else /* __GNUC_AS__ */ 3522 mov %cs, %edx 3523 #endif /* __GNUC_AS__ */ 3524 movl %edx, REGOFF_CS(%esp) 3525 pushfl 3526 popl %ecx 3527 #if defined(__xpv) 3528 /* 3529 * Synthesize the PS_IE bit from the event mask bit 3530 */ 3531 CURTHREAD(%edx) 3532 KPREEMPT_DISABLE(%edx) 3533 EVENT_MASK_TO_IE(%edx, %ecx) 3534 CURTHREAD(%edx) 3535 KPREEMPT_ENABLE_NOKP(%edx) 3536 #endif 3537 movl %ecx, REGOFF_EFL(%esp) 3538 movl $0, REGOFF_UESP(%esp) 3539 #if !defined(__GNUC_AS__) 3540 movw %ss, %edx 3541 #else /* __GNUC_AS__ */ 3542 mov %ss, %edx 3543 #endif /* __GNUC_AS__ */ 3544 movl %edx, REGOFF_SS(%esp) 3545 3546 movl %esp, %ecx / %ecx = ®s 3547 pushl %eax / push on_panic_stack 3548 pushl %ecx / push ®s 3549 movl 12(%ebp), %ecx / %ecx = alist 3550 pushl %ecx / push alist 3551 movl 8(%ebp), %ecx / %ecx = format 3552 pushl %ecx / push format 3553 call panicsys / panicsys(); 3554 addl $16, %esp / pop arguments 3555 3556 addl $REGSIZE, %esp 3557 popl %edx 3558 popl %ecx 3559 popl %ebx 3560 popl %eax 3561 leave 3562 ret 3563 SET_SIZE(vpanic) 3564 3565 ENTRY_NP(dtrace_vpanic) / Initial stack layout: 3566 3567 pushl %ebp / | %eip | 20 3568 movl %esp, %ebp / | %ebp | 16 3569 pushl %eax / | %eax | 12 3570 pushl %ebx / | %ebx | 8 3571 pushl %ecx / | %ecx | 4 3572 pushl %edx / | %edx | 0 3573 3574 movl %esp, %ebx / %ebx = current stack pointer 3575 3576 lea panic_quiesce, %eax / %eax = &panic_quiesce 3577 pushl %eax / push &panic_quiesce 3578 call dtrace_panic_trigger / %eax = dtrace_panic_trigger() 3579 addl $4, %esp / reset stack pointer 3580 jmp vpanic_common / jump back to common code 3581 3582 SET_SIZE(dtrace_vpanic) 3583 3584 #endif /* __i386 */ 3585 #endif /* __lint */ 3586 3587 #if defined(__lint) 3588 3589 void 3590 hres_tick(void) 3591 {} 3592 3593 int64_t timedelta; 3594 hrtime_t hrtime_base; 3595 3596 #else /* __lint */ 3597 3598 DGDEF3(timedelta, 8, 8) 3599 .long 0, 0 3600 3601 /* 3602 * initialized to a non zero value to make pc_gethrtime() 3603 * work correctly even before clock is initialized 3604 */ 3605 DGDEF3(hrtime_base, 8, 8) 3606 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 3607 3608 DGDEF3(adj_shift, 4, 4) 3609 .long ADJ_SHIFT 3610 3611 #if defined(__amd64) 3612 3613 ENTRY_NP(hres_tick) 3614 pushq %rbp 3615 movq %rsp, %rbp 3616 3617 /* 3618 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3619 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3620 * At worst, performing this now instead of under CLOCK_LOCK may 3621 * introduce some jitter in pc_gethrestime(). 3622 */ 3623 call *gethrtimef(%rip) 3624 movq %rax, %r8 3625 3626 leaq hres_lock(%rip), %rax 3627 movb $-1, %dl 3628 .CL1: 3629 xchgb %dl, (%rax) 3630 testb %dl, %dl 3631 jz .CL3 /* got it */ 3632 .CL2: 3633 cmpb $0, (%rax) /* possible to get lock? */ 3634 pause 3635 jne .CL2 3636 jmp .CL1 /* yes, try again */ 3637 .CL3: 3638 /* 3639 * compute the interval since last time hres_tick was called 3640 * and adjust hrtime_base and hrestime accordingly 3641 * hrtime_base is an 8 byte value (in nsec), hrestime is 3642 * a timestruc_t (sec, nsec) 3643 */ 3644 leaq hres_last_tick(%rip), %rax 3645 movq %r8, %r11 3646 subq (%rax), %r8 3647 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 3648 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 3649 /* 3650 * Now that we have CLOCK_LOCK, we can update hres_last_tick 3651 */ 3652 movq %r11, (%rax) 3653 3654 call __adj_hrestime 3655 3656 /* 3657 * release the hres_lock 3658 */ 3659 incl hres_lock(%rip) 3660 leave 3661 ret 3662 SET_SIZE(hres_tick) 3663 3664 #elif defined(__i386) 3665 3666 ENTRY_NP(hres_tick) 3667 pushl %ebp 3668 movl %esp, %ebp 3669 pushl %esi 3670 pushl %ebx 3671 3672 /* 3673 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3674 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3675 * At worst, performing this now instead of under CLOCK_LOCK may 3676 * introduce some jitter in pc_gethrestime(). 3677 */ 3678 call *gethrtimef 3679 movl %eax, %ebx 3680 movl %edx, %esi 3681 3682 movl $hres_lock, %eax 3683 movl $-1, %edx 3684 .CL1: 3685 xchgb %dl, (%eax) 3686 testb %dl, %dl 3687 jz .CL3 / got it 3688 .CL2: 3689 cmpb $0, (%eax) / possible to get lock? 3690 pause 3691 jne .CL2 3692 jmp .CL1 / yes, try again 3693 .CL3: 3694 /* 3695 * compute the interval since last time hres_tick was called 3696 * and adjust hrtime_base and hrestime accordingly 3697 * hrtime_base is an 8 byte value (in nsec), hrestime is 3698 * timestruc_t (sec, nsec) 3699 */ 3700 3701 lea hres_last_tick, %eax 3702 3703 movl %ebx, %edx 3704 movl %esi, %ecx 3705 3706 subl (%eax), %edx 3707 sbbl 4(%eax), %ecx 3708 3709 addl %edx, hrtime_base / add interval to hrtime_base 3710 adcl %ecx, hrtime_base+4 3711 3712 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec 3713 3714 / 3715 / Now that we have CLOCK_LOCK, we can update hres_last_tick. 3716 / 3717 movl %ebx, (%eax) 3718 movl %esi, 4(%eax) 3719 3720 / get hrestime at this moment. used as base for pc_gethrestime 3721 / 3722 / Apply adjustment, if any 3723 / 3724 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT) 3725 / (max_hres_adj) 3726 / 3727 / void 3728 / adj_hrestime() 3729 / { 3730 / long long adj; 3731 / 3732 / if (hrestime_adj == 0) 3733 / adj = 0; 3734 / else if (hrestime_adj > 0) { 3735 / if (hrestime_adj < HRES_ADJ) 3736 / adj = hrestime_adj; 3737 / else 3738 / adj = HRES_ADJ; 3739 / } 3740 / else { 3741 / if (hrestime_adj < -(HRES_ADJ)) 3742 / adj = -(HRES_ADJ); 3743 / else 3744 / adj = hrestime_adj; 3745 / } 3746 / 3747 / timedelta -= adj; 3748 / hrestime_adj = timedelta; 3749 / hrestime.tv_nsec += adj; 3750 / 3751 / while (hrestime.tv_nsec >= NANOSEC) { 3752 / one_sec++; 3753 / hrestime.tv_sec++; 3754 / hrestime.tv_nsec -= NANOSEC; 3755 / } 3756 / } 3757 __adj_hrestime: 3758 movl hrestime_adj, %esi / if (hrestime_adj == 0) 3759 movl hrestime_adj+4, %edx 3760 andl %esi, %esi 3761 jne .CL4 / no 3762 andl %edx, %edx 3763 jne .CL4 / no 3764 subl %ecx, %ecx / yes, adj = 0; 3765 subl %edx, %edx 3766 jmp .CL5 3767 .CL4: 3768 subl %ecx, %ecx 3769 subl %eax, %eax 3770 subl %esi, %ecx 3771 sbbl %edx, %eax 3772 andl %eax, %eax / if (hrestime_adj > 0) 3773 jge .CL6 3774 3775 / In the following comments, HRES_ADJ is used, while in the code 3776 / max_hres_adj is used. 3777 / 3778 / The test for "hrestime_adj < HRES_ADJ" is complicated because 3779 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3780 / on the logical equivalence of: 3781 / 3782 / !(hrestime_adj < HRES_ADJ) 3783 / 3784 / and the two step sequence: 3785 / 3786 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry 3787 / 3788 / which computes whether or not the least significant 32-bits 3789 / of hrestime_adj is greater than HRES_ADJ, followed by: 3790 / 3791 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry 3792 / 3793 / which generates a carry whenever step 1 is true or the most 3794 / significant long of the longlong hrestime_adj is non-zero. 3795 3796 movl max_hres_adj, %ecx / hrestime_adj is positive 3797 subl %esi, %ecx 3798 movl %edx, %eax 3799 adcl $-1, %eax 3800 jnc .CL7 3801 movl max_hres_adj, %ecx / adj = HRES_ADJ; 3802 subl %edx, %edx 3803 jmp .CL5 3804 3805 / The following computation is similar to the one above. 3806 / 3807 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because 3808 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3809 / on the logical equivalence of: 3810 / 3811 / (hrestime_adj > -HRES_ADJ) 3812 / 3813 / and the two step sequence: 3814 / 3815 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry 3816 / 3817 / which means the least significant 32-bits of hrestime_adj is 3818 / greater than -HRES_ADJ, followed by: 3819 / 3820 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry 3821 / 3822 / which generates a carry only when step 1 is true and the most 3823 / significant long of the longlong hrestime_adj is -1. 3824 3825 .CL6: / hrestime_adj is negative 3826 movl %esi, %ecx 3827 addl max_hres_adj, %ecx 3828 movl %edx, %eax 3829 adcl $0, %eax 3830 jc .CL7 3831 xor %ecx, %ecx 3832 subl max_hres_adj, %ecx / adj = -(HRES_ADJ); 3833 movl $-1, %edx 3834 jmp .CL5 3835 .CL7: 3836 movl %esi, %ecx / adj = hrestime_adj; 3837 .CL5: 3838 movl timedelta, %esi 3839 subl %ecx, %esi 3840 movl timedelta+4, %eax 3841 sbbl %edx, %eax 3842 movl %esi, timedelta 3843 movl %eax, timedelta+4 / timedelta -= adj; 3844 movl %esi, hrestime_adj 3845 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta; 3846 addl hrestime+4, %ecx 3847 3848 movl %ecx, %eax / eax = tv_nsec 3849 1: 3850 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC) 3851 jb .CL8 / no 3852 incl one_sec / yes, one_sec++; 3853 incl hrestime / hrestime.tv_sec++; 3854 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC 3855 jmp 1b / check for more seconds 3856 3857 .CL8: 3858 movl %eax, hrestime+4 / store final into hrestime.tv_nsec 3859 incl hres_lock / release the hres_lock 3860 3861 popl %ebx 3862 popl %esi 3863 leave 3864 ret 3865 SET_SIZE(hres_tick) 3866 3867 #endif /* __i386 */ 3868 #endif /* __lint */ 3869 3870 /* 3871 * void prefetch_smap_w(void *) 3872 * 3873 * Prefetch ahead within a linear list of smap structures. 3874 * Not implemented for ia32. Stub for compatibility. 3875 */ 3876 3877 #if defined(__lint) 3878 3879 /*ARGSUSED*/ 3880 void prefetch_smap_w(void *smp) 3881 {} 3882 3883 #else /* __lint */ 3884 3885 ENTRY(prefetch_smap_w) 3886 rep; ret /* use 2 byte return instruction when branch target */ 3887 /* AMD Software Optimization Guide - Section 6.2 */ 3888 SET_SIZE(prefetch_smap_w) 3889 3890 #endif /* __lint */ 3891 3892 /* 3893 * prefetch_page_r(page_t *) 3894 * issue prefetch instructions for a page_t 3895 */ 3896 #if defined(__lint) 3897 3898 /*ARGSUSED*/ 3899 void 3900 prefetch_page_r(void *pp) 3901 {} 3902 3903 #else /* __lint */ 3904 3905 ENTRY(prefetch_page_r) 3906 rep; ret /* use 2 byte return instruction when branch target */ 3907 /* AMD Software Optimization Guide - Section 6.2 */ 3908 SET_SIZE(prefetch_page_r) 3909 3910 #endif /* __lint */ 3911 3912 #if defined(__lint) 3913 3914 /*ARGSUSED*/ 3915 int 3916 bcmp(const void *s1, const void *s2, size_t count) 3917 { return (0); } 3918 3919 #else /* __lint */ 3920 3921 #if defined(__amd64) 3922 3923 ENTRY(bcmp) 3924 pushq %rbp 3925 movq %rsp, %rbp 3926 #ifdef DEBUG 3927 testq %rdx,%rdx 3928 je 1f 3929 movq postbootkernelbase(%rip), %r11 3930 cmpq %r11, %rdi 3931 jb 0f 3932 cmpq %r11, %rsi 3933 jnb 1f 3934 0: leaq .bcmp_panic_msg(%rip), %rdi 3935 xorl %eax, %eax 3936 call panic 3937 1: 3938 #endif /* DEBUG */ 3939 call memcmp 3940 testl %eax, %eax 3941 setne %dl 3942 leave 3943 movzbl %dl, %eax 3944 ret 3945 SET_SIZE(bcmp) 3946 3947 #elif defined(__i386) 3948 3949 #define ARG_S1 8 3950 #define ARG_S2 12 3951 #define ARG_LENGTH 16 3952 3953 ENTRY(bcmp) 3954 pushl %ebp 3955 movl %esp, %ebp / create new stack frame 3956 #ifdef DEBUG 3957 cmpl $0, ARG_LENGTH(%ebp) 3958 je 1f 3959 movl postbootkernelbase, %eax 3960 cmpl %eax, ARG_S1(%ebp) 3961 jb 0f 3962 cmpl %eax, ARG_S2(%ebp) 3963 jnb 1f 3964 0: pushl $.bcmp_panic_msg 3965 call panic 3966 1: 3967 #endif /* DEBUG */ 3968 3969 pushl %edi / save register variable 3970 movl ARG_S1(%ebp), %eax / %eax = address of string 1 3971 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2 3972 cmpl %eax, %ecx / if the same string 3973 je .equal / goto .equal 3974 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes 3975 cmpl $4, %edi / if %edi < 4 3976 jb .byte_check / goto .byte_check 3977 .align 4 3978 .word_loop: 3979 movl (%ecx), %edx / move 1 word from (%ecx) to %edx 3980 leal -4(%edi), %edi / %edi -= 4 3981 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx 3982 jne .word_not_equal / if not equal, goto .word_not_equal 3983 leal 4(%ecx), %ecx / %ecx += 4 (next word) 3984 leal 4(%eax), %eax / %eax += 4 (next word) 3985 cmpl $4, %edi / if %edi >= 4 3986 jae .word_loop / goto .word_loop 3987 .byte_check: 3988 cmpl $0, %edi / if %edi == 0 3989 je .equal / goto .equal 3990 jmp .byte_loop / goto .byte_loop (checks in bytes) 3991 .word_not_equal: 3992 leal 4(%edi), %edi / %edi += 4 (post-decremented) 3993 .align 4 3994 .byte_loop: 3995 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl 3996 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax) 3997 jne .not_equal / if not equal, goto .not_equal 3998 incl %ecx / %ecx++ (next byte) 3999 incl %eax / %eax++ (next byte) 4000 decl %edi / %edi-- 4001 jnz .byte_loop / if not zero, goto .byte_loop 4002 .equal: 4003 xorl %eax, %eax / %eax = 0 4004 popl %edi / restore register variable 4005 leave / restore old stack frame 4006 ret / return (NULL) 4007 .align 4 4008 .not_equal: 4009 movl $1, %eax / return 1 4010 popl %edi / restore register variable 4011 leave / restore old stack frame 4012 ret / return (NULL) 4013 SET_SIZE(bcmp) 4014 4015 #endif /* __i386 */ 4016 4017 #ifdef DEBUG 4018 .text 4019 .bcmp_panic_msg: 4020 .string "bcmp: arguments below kernelbase" 4021 #endif /* DEBUG */ 4022 4023 #endif /* __lint */ 4024 4025 #if defined(__lint) 4026 4027 uint_t 4028 bsrw_insn(uint16_t mask) 4029 { 4030 uint_t index = sizeof (mask) * NBBY - 1; 4031 4032 while ((mask & (1 << index)) == 0) 4033 index--; 4034 return (index); 4035 } 4036 4037 #else /* __lint */ 4038 4039 #if defined(__amd64) 4040 4041 ENTRY_NP(bsrw_insn) 4042 xorl %eax, %eax 4043 bsrw %di, %ax 4044 ret 4045 SET_SIZE(bsrw_insn) 4046 4047 #elif defined(__i386) 4048 4049 ENTRY_NP(bsrw_insn) 4050 movw 4(%esp), %cx 4051 xorl %eax, %eax 4052 bsrw %cx, %ax 4053 ret 4054 SET_SIZE(bsrw_insn) 4055 4056 #endif /* __i386 */ 4057 #endif /* __lint */ 4058 4059 #if defined(__lint) 4060 4061 uint_t 4062 atomic_btr32(uint32_t *pending, uint_t pil) 4063 { 4064 return (*pending &= ~(1 << pil)); 4065 } 4066 4067 #else /* __lint */ 4068 4069 #if defined(__i386) 4070 4071 ENTRY_NP(atomic_btr32) 4072 movl 4(%esp), %ecx 4073 movl 8(%esp), %edx 4074 xorl %eax, %eax 4075 lock 4076 btrl %edx, (%ecx) 4077 setc %al 4078 ret 4079 SET_SIZE(atomic_btr32) 4080 4081 #endif /* __i386 */ 4082 #endif /* __lint */ 4083 4084 #if defined(__lint) 4085 4086 /*ARGSUSED*/ 4087 void 4088 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1, 4089 uint_t arg2) 4090 {} 4091 4092 #else /* __lint */ 4093 4094 #if defined(__amd64) 4095 4096 ENTRY_NP(switch_sp_and_call) 4097 pushq %rbp 4098 movq %rsp, %rbp /* set up stack frame */ 4099 movq %rdi, %rsp /* switch stack pointer */ 4100 movq %rdx, %rdi /* pass func arg 1 */ 4101 movq %rsi, %r11 /* save function to call */ 4102 movq %rcx, %rsi /* pass func arg 2 */ 4103 call *%r11 /* call function */ 4104 leave /* restore stack */ 4105 ret 4106 SET_SIZE(switch_sp_and_call) 4107 4108 #elif defined(__i386) 4109 4110 ENTRY_NP(switch_sp_and_call) 4111 pushl %ebp 4112 mov %esp, %ebp /* set up stack frame */ 4113 movl 8(%ebp), %esp /* switch stack pointer */ 4114 pushl 20(%ebp) /* push func arg 2 */ 4115 pushl 16(%ebp) /* push func arg 1 */ 4116 call *12(%ebp) /* call function */ 4117 addl $8, %esp /* pop arguments */ 4118 leave /* restore stack */ 4119 ret 4120 SET_SIZE(switch_sp_and_call) 4121 4122 #endif /* __i386 */ 4123 #endif /* __lint */ 4124 4125 #if defined(__lint) 4126 4127 void 4128 kmdb_enter(void) 4129 {} 4130 4131 #else /* __lint */ 4132 4133 #if defined(__amd64) 4134 4135 ENTRY_NP(kmdb_enter) 4136 pushq %rbp 4137 movq %rsp, %rbp 4138 4139 /* 4140 * Save flags, do a 'cli' then return the saved flags 4141 */ 4142 call intr_clear 4143 4144 int $T_DBGENTR 4145 4146 /* 4147 * Restore the saved flags 4148 */ 4149 movq %rax, %rdi 4150 call intr_restore 4151 4152 leave 4153 ret 4154 SET_SIZE(kmdb_enter) 4155 4156 #elif defined(__i386) 4157 4158 ENTRY_NP(kmdb_enter) 4159 pushl %ebp 4160 movl %esp, %ebp 4161 4162 /* 4163 * Save flags, do a 'cli' then return the saved flags 4164 */ 4165 call intr_clear 4166 4167 int $T_DBGENTR 4168 4169 /* 4170 * Restore the saved flags 4171 */ 4172 pushl %eax 4173 call intr_restore 4174 addl $4, %esp 4175 4176 leave 4177 ret 4178 SET_SIZE(kmdb_enter) 4179 4180 #endif /* __i386 */ 4181 #endif /* __lint */ 4182 4183 #if defined(__lint) 4184 4185 void 4186 return_instr(void) 4187 {} 4188 4189 #else /* __lint */ 4190 4191 ENTRY_NP(return_instr) 4192 rep; ret /* use 2 byte instruction when branch target */ 4193 /* AMD Software Optimization Guide - Section 6.2 */ 4194 SET_SIZE(return_instr) 4195 4196 #endif /* __lint */ 4197 4198 #if defined(__lint) 4199 4200 ulong_t 4201 getflags(void) 4202 { 4203 return (0); 4204 } 4205 4206 #else /* __lint */ 4207 4208 #if defined(__amd64) 4209 4210 ENTRY(getflags) 4211 pushfq 4212 popq %rax 4213 #if defined(__xpv) 4214 CURTHREAD(%rdi) 4215 KPREEMPT_DISABLE(%rdi) 4216 /* 4217 * Synthesize the PS_IE bit from the event mask bit 4218 */ 4219 CURVCPU(%r11) 4220 andq $_BITNOT(PS_IE), %rax 4221 XEN_TEST_UPCALL_MASK(%r11) 4222 jnz 1f 4223 orq $PS_IE, %rax 4224 1: 4225 KPREEMPT_ENABLE_NOKP(%rdi) 4226 #endif 4227 ret 4228 SET_SIZE(getflags) 4229 4230 #elif defined(__i386) 4231 4232 ENTRY(getflags) 4233 pushfl 4234 popl %eax 4235 #if defined(__xpv) 4236 CURTHREAD(%ecx) 4237 KPREEMPT_DISABLE(%ecx) 4238 /* 4239 * Synthesize the PS_IE bit from the event mask bit 4240 */ 4241 CURVCPU(%edx) 4242 andl $_BITNOT(PS_IE), %eax 4243 XEN_TEST_UPCALL_MASK(%edx) 4244 jnz 1f 4245 orl $PS_IE, %eax 4246 1: 4247 KPREEMPT_ENABLE_NOKP(%ecx) 4248 #endif 4249 ret 4250 SET_SIZE(getflags) 4251 4252 #endif /* __i386 */ 4253 4254 #endif /* __lint */ 4255 4256 #if defined(__lint) 4257 4258 ftrace_icookie_t 4259 ftrace_interrupt_disable(void) 4260 { return (0); } 4261 4262 #else /* __lint */ 4263 4264 #if defined(__amd64) 4265 4266 ENTRY(ftrace_interrupt_disable) 4267 pushfq 4268 popq %rax 4269 CLI(%rdx) 4270 ret 4271 SET_SIZE(ftrace_interrupt_disable) 4272 4273 #elif defined(__i386) 4274 4275 ENTRY(ftrace_interrupt_disable) 4276 pushfl 4277 popl %eax 4278 CLI(%edx) 4279 ret 4280 SET_SIZE(ftrace_interrupt_disable) 4281 4282 #endif /* __i386 */ 4283 #endif /* __lint */ 4284 4285 #if defined(__lint) 4286 4287 /*ARGSUSED*/ 4288 void 4289 ftrace_interrupt_enable(ftrace_icookie_t cookie) 4290 {} 4291 4292 #else /* __lint */ 4293 4294 #if defined(__amd64) 4295 4296 ENTRY(ftrace_interrupt_enable) 4297 pushq %rdi 4298 popfq 4299 ret 4300 SET_SIZE(ftrace_interrupt_enable) 4301 4302 #elif defined(__i386) 4303 4304 ENTRY(ftrace_interrupt_enable) 4305 movl 4(%esp), %eax 4306 pushl %eax 4307 popfl 4308 ret 4309 SET_SIZE(ftrace_interrupt_enable) 4310 4311 #endif /* __i386 */ 4312 #endif /* __lint */ 4313 4314 #if defined (__lint) 4315 4316 /*ARGSUSED*/ 4317 void 4318 clflush_insn(caddr_t addr) 4319 {} 4320 4321 #else /* __lint */ 4322 4323 #if defined (__amd64) 4324 ENTRY(clflush_insn) 4325 clflush (%rdi) 4326 ret 4327 SET_SIZE(clflush_insn) 4328 #elif defined (__i386) 4329 ENTRY(clflush_insn) 4330 movl 4(%esp), %eax 4331 clflush (%eax) 4332 ret 4333 SET_SIZE(clflush_insn) 4334 4335 #endif /* __i386 */ 4336 #endif /* __lint */ 4337 4338 #if defined (__lint) 4339 /*ARGSUSED*/ 4340 void 4341 mfence_insn(void) 4342 {} 4343 4344 #else /* __lint */ 4345 4346 #if defined (__amd64) 4347 ENTRY(mfence_insn) 4348 mfence 4349 ret 4350 SET_SIZE(mfence_insn) 4351 #elif defined (__i386) 4352 ENTRY(mfence_insn) 4353 mfence 4354 ret 4355 SET_SIZE(mfence_insn) 4356 4357 #endif /* __i386 */ 4358 #endif /* __lint */ 4359 4360 /* 4361 * VMware implements an I/O port that programs can query to detect if software 4362 * is running in a VMware hypervisor. This hypervisor port behaves differently 4363 * depending on magic values in certain registers and modifies some registers 4364 * as a side effect. 4365 * 4366 * References: http://kb.vmware.com/kb/1009458 4367 */ 4368 4369 #if defined(__lint) 4370 4371 /* ARGSUSED */ 4372 void 4373 vmware_port(int cmd, uint32_t *regs) { return; } 4374 4375 #else 4376 4377 #if defined(__amd64) 4378 4379 ENTRY(vmware_port) 4380 pushq %rbx 4381 movl $VMWARE_HVMAGIC, %eax 4382 movl $0xffffffff, %ebx 4383 movl %edi, %ecx 4384 movl $VMWARE_HVPORT, %edx 4385 inl (%dx) 4386 movl %eax, (%rsi) 4387 movl %ebx, 4(%rsi) 4388 movl %ecx, 8(%rsi) 4389 movl %edx, 12(%rsi) 4390 popq %rbx 4391 ret 4392 SET_SIZE(vmware_port) 4393 4394 #elif defined(__i386) 4395 4396 ENTRY(vmware_port) 4397 pushl %ebx 4398 pushl %esi 4399 movl $VMWARE_HVMAGIC, %eax 4400 movl $0xffffffff, %ebx 4401 movl 12(%esp), %ecx 4402 movl $VMWARE_HVPORT, %edx 4403 inl (%dx) 4404 movl 16(%esp), %esi 4405 movl %eax, (%esi) 4406 movl %ebx, 4(%esi) 4407 movl %ecx, 8(%esi) 4408 movl %edx, 12(%esi) 4409 popl %esi 4410 popl %ebx 4411 ret 4412 SET_SIZE(vmware_port) 4413 4414 #endif /* __i386 */ 4415 #endif /* __lint */