40 /*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
57 #include <sys/psw.h>
58 #include <sys/x86_archext.h>
59
60 #if defined(__lint)
61 #include <sys/types.h>
62 #include <sys/systm.h>
63 #include <sys/thread.h>
64 #include <sys/archsystm.h>
65 #include <sys/byteorder.h>
66 #include <sys/dtrace.h>
67 #include <sys/ftrace.h>
68 #else /* __lint */
69 #include "assym.h"
70 #endif /* __lint */
71 #include <sys/dditypes.h>
72
73 /*
74 * on_fault()
75 *
76 * Catch lofault faults. Like setjmp except it returns one
77 * if code following causes uncorrectable fault. Turned off
78 * by calling no_fault(). Note that while under on_fault(),
79 * SMAP is disabled. For more information see
80 * uts/intel/ia32/ml/copy.s.
81 */
82
83 #if defined(__lint)
84
85 /* ARGSUSED */
86 int
87 on_fault(label_t *ljb)
88 { return (0); }
89
90 void
91 no_fault(void)
92 {}
93
94 #else /* __lint */
95
96 #if defined(__amd64)
97
98 ENTRY(on_fault)
99 movq %gs:CPU_THREAD, %rsi
100 leaq catch_fault(%rip), %rdx
101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
103 call smap_disable /* allow user accesses */
104 jmp setjmp /* let setjmp do the rest */
105
106 catch_fault:
107 movq %gs:CPU_THREAD, %rsi
108 movq T_ONFAULT(%rsi), %rdi /* address of save area */
109 xorl %eax, %eax
110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
112 call smap_enable /* disallow user accesses */
113 jmp longjmp /* let longjmp do the rest */
114 SET_SIZE(on_fault)
115
116 ENTRY(no_fault)
117 movq %gs:CPU_THREAD, %rsi
118 xorl %eax, %eax
119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
121 call smap_enable /* disallow user accesses */
122 ret
123 SET_SIZE(no_fault)
124
125 #elif defined(__i386)
126
127 ENTRY(on_fault)
128 movl %gs:CPU_THREAD, %edx
129 movl 4(%esp), %eax /* jumpbuf address */
130 leal catch_fault, %ecx
131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
133 jmp setjmp /* let setjmp do the rest */
134
135 catch_fault:
136 movl %gs:CPU_THREAD, %edx
137 xorl %eax, %eax
138 movl T_ONFAULT(%edx), %ecx /* address of save area */
139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
141 pushl %ecx
142 call longjmp /* let longjmp do the rest */
143 SET_SIZE(on_fault)
144
145 ENTRY(no_fault)
146 movl %gs:CPU_THREAD, %edx
147 xorl %eax, %eax
148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
150 ret
151 SET_SIZE(no_fault)
152
153 #endif /* __i386 */
154 #endif /* __lint */
155
156 /*
157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 */
160
161 #if defined(lint)
162
163 void
164 on_trap_trampoline(void)
165 {}
166
167 #else /* __lint */
168
169 #if defined(__amd64)
170
171 ENTRY(on_trap_trampoline)
172 movq %gs:CPU_THREAD, %rsi
173 movq T_ONTRAP(%rsi), %rdi
174 addq $OT_JMPBUF, %rdi
175 jmp longjmp
176 SET_SIZE(on_trap_trampoline)
177
178 #elif defined(__i386)
179
180 ENTRY(on_trap_trampoline)
181 movl %gs:CPU_THREAD, %eax
182 movl T_ONTRAP(%eax), %eax
183 addl $OT_JMPBUF, %eax
184 pushl %eax
185 call longjmp
186 SET_SIZE(on_trap_trampoline)
187
188 #endif /* __i386 */
189 #endif /* __lint */
190
191 /*
192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
193 * more information about the on_trap() mechanism. If the on_trap_data is the
194 * same as the topmost stack element, we just modify that element.
195 */
196 #if defined(lint)
197
198 /*ARGSUSED*/
199 int
200 on_trap(on_trap_data_t *otp, uint_t prot)
201 { return (0); }
202
203 #else /* __lint */
204
205 #if defined(__amd64)
206
207 ENTRY(on_trap)
208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
212 xorl %ecx, %ecx
213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
217 cmpq %rdi, %rcx /* if (otp == %rcx) */
218 je 0f /* don't modify t_ontrap */
219
220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
222
223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
224 jmp setjmp
225 SET_SIZE(on_trap)
226
227 #elif defined(__i386)
228
229 ENTRY(on_trap)
230 movl 4(%esp), %eax /* %eax = otp */
231 movl 8(%esp), %edx /* %edx = prot */
232
233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
241 cmpl %eax, %ecx /* if (otp == %ecx) */
242 je 0f /* don't modify t_ontrap */
243
244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
246
247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
248 movl %eax, 4(%esp) /* put %eax back on the stack */
249 jmp setjmp /* let setjmp do the rest */
250 SET_SIZE(on_trap)
251
252 #endif /* __i386 */
253 #endif /* __lint */
254
255 /*
256 * Setjmp and longjmp implement non-local gotos using state vectors
257 * type label_t.
258 */
259
260 #if defined(__lint)
261
262 /* ARGSUSED */
263 int
264 setjmp(label_t *lp)
265 { return (0); }
266
267 /* ARGSUSED */
268 void
269 longjmp(label_t *lp)
270 {}
271
272 #else /* __lint */
273
274 #if LABEL_PC != 0
275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276 #endif /* LABEL_PC != 0 */
277
278 #if defined(__amd64)
279
280 ENTRY(setjmp)
281 movq %rsp, LABEL_SP(%rdi)
282 movq %rbp, LABEL_RBP(%rdi)
283 movq %rbx, LABEL_RBX(%rdi)
284 movq %r12, LABEL_R12(%rdi)
285 movq %r13, LABEL_R13(%rdi)
286 movq %r14, LABEL_R14(%rdi)
287 movq %r15, LABEL_R15(%rdi)
288 movq (%rsp), %rdx /* return address */
289 movq %rdx, (%rdi) /* LABEL_PC is 0 */
290 xorl %eax, %eax /* return 0 */
291 ret
292 SET_SIZE(setjmp)
293
294 ENTRY(longjmp)
295 movq LABEL_SP(%rdi), %rsp
296 movq LABEL_RBP(%rdi), %rbp
297 movq LABEL_RBX(%rdi), %rbx
298 movq LABEL_R12(%rdi), %r12
299 movq LABEL_R13(%rdi), %r13
300 movq LABEL_R14(%rdi), %r14
301 movq LABEL_R15(%rdi), %r15
302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
303 movq %rdx, (%rsp)
304 xorl %eax, %eax
305 incl %eax /* return 1 */
306 ret
307 SET_SIZE(longjmp)
308
309 #elif defined(__i386)
310
311 ENTRY(setjmp)
312 movl 4(%esp), %edx /* address of save area */
313 movl %ebp, LABEL_EBP(%edx)
314 movl %ebx, LABEL_EBX(%edx)
315 movl %esi, LABEL_ESI(%edx)
316 movl %edi, LABEL_EDI(%edx)
317 movl %esp, 4(%edx)
318 movl (%esp), %ecx /* %eip (return address) */
319 movl %ecx, (%edx) /* LABEL_PC is 0 */
320 subl %eax, %eax /* return 0 */
321 ret
322 SET_SIZE(setjmp)
323
324 ENTRY(longjmp)
325 movl 4(%esp), %edx /* address of save area */
326 movl LABEL_EBP(%edx), %ebp
327 movl LABEL_EBX(%edx), %ebx
328 movl LABEL_ESI(%edx), %esi
329 movl LABEL_EDI(%edx), %edi
330 movl 4(%edx), %esp
331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
332 movl $1, %eax
333 addl $4, %esp /* pop ret adr */
334 jmp *%ecx /* indirect */
335 SET_SIZE(longjmp)
336
337 #endif /* __i386 */
338 #endif /* __lint */
339
340 /*
341 * if a() calls b() calls caller(),
342 * caller() returns return address in a().
343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 * sequence.)
345 */
346
347 #if defined(__lint)
348
349 caddr_t
350 caller(void)
351 { return (0); }
352
353 #else /* __lint */
354
355 #if defined(__amd64)
356
357 ENTRY(caller)
358 movq 8(%rbp), %rax /* b()'s return pc, in a() */
359 ret
360 SET_SIZE(caller)
361
362 #elif defined(__i386)
363
364 ENTRY(caller)
365 movl 4(%ebp), %eax /* b()'s return pc, in a() */
366 ret
367 SET_SIZE(caller)
368
369 #endif /* __i386 */
370 #endif /* __lint */
371
372 /*
373 * if a() calls callee(), callee() returns the
374 * return address in a();
375 */
376
377 #if defined(__lint)
378
379 caddr_t
380 callee(void)
381 { return (0); }
382
383 #else /* __lint */
384
385 #if defined(__amd64)
386
387 ENTRY(callee)
388 movq (%rsp), %rax /* callee()'s return pc, in a() */
389 ret
390 SET_SIZE(callee)
391
392 #elif defined(__i386)
393
394 ENTRY(callee)
395 movl (%esp), %eax /* callee()'s return pc, in a() */
396 ret
397 SET_SIZE(callee)
398
399 #endif /* __i386 */
400 #endif /* __lint */
401
402 /*
403 * return the current frame pointer
404 */
405
406 #if defined(__lint)
407
408 greg_t
409 getfp(void)
410 { return (0); }
411
412 #else /* __lint */
413
414 #if defined(__amd64)
415
416 ENTRY(getfp)
417 movq %rbp, %rax
418 ret
419 SET_SIZE(getfp)
420
421 #elif defined(__i386)
422
423 ENTRY(getfp)
424 movl %ebp, %eax
425 ret
426 SET_SIZE(getfp)
427
428 #endif /* __i386 */
429 #endif /* __lint */
430
431 /*
432 * Invalidate a single page table entry in the TLB
433 */
434
435 #if defined(__lint)
436
437 /* ARGSUSED */
438 void
439 mmu_invlpg(caddr_t m)
440 {}
441
442 #else /* __lint */
443
444 ENTRY(mmu_invlpg)
445 invlpg (%rdi)
446 ret
447 SET_SIZE(mmu_invlpg)
448
449 #endif /* __lint */
450
451
452 /*
453 * Get/Set the value of various control registers
454 */
455
456 #if defined(__lint)
457
458 ulong_t
459 getcr0(void)
460 { return (0); }
461
462 /* ARGSUSED */
463 void
464 setcr0(ulong_t value)
465 {}
466
467 ulong_t
468 getcr2(void)
469 { return (0); }
470
471 ulong_t
472 getcr3(void)
473 { return (0); }
474
475 #if !defined(__xpv)
476 /* ARGSUSED */
477 void
478 setcr3(ulong_t val)
479 {}
480
481 void
482 reload_cr3(void)
483 {}
484 #endif
485
486 ulong_t
487 getcr4(void)
488 { return (0); }
489
490 /* ARGSUSED */
491 void
492 setcr4(ulong_t val)
493 {}
494
495 #if defined(__amd64)
496
497 ulong_t
498 getcr8(void)
499 { return (0); }
500
501 /* ARGSUSED */
502 void
503 setcr8(ulong_t val)
504 {}
505
506 #endif /* __amd64 */
507
508 #else /* __lint */
509
510 #if defined(__amd64)
511
512 ENTRY(getcr0)
513 movq %cr0, %rax
514 ret
515 SET_SIZE(getcr0)
516
517 ENTRY(setcr0)
518 movq %rdi, %cr0
519 ret
520 SET_SIZE(setcr0)
521
522 ENTRY(getcr2)
523 #if defined(__xpv)
524 movq %gs:CPU_VCPU_INFO, %rax
525 movq VCPU_INFO_ARCH_CR2(%rax), %rax
526 #else
527 movq %cr2, %rax
528 #endif
529 ret
530 SET_SIZE(getcr2)
531
552 ENTRY(getcr4)
553 movq %cr4, %rax
554 ret
555 SET_SIZE(getcr4)
556
557 ENTRY(setcr4)
558 movq %rdi, %cr4
559 ret
560 SET_SIZE(setcr4)
561
562 ENTRY(getcr8)
563 movq %cr8, %rax
564 ret
565 SET_SIZE(getcr8)
566
567 ENTRY(setcr8)
568 movq %rdi, %cr8
569 ret
570 SET_SIZE(setcr8)
571
572 #elif defined(__i386)
573
574 ENTRY(getcr0)
575 movl %cr0, %eax
576 ret
577 SET_SIZE(getcr0)
578
579 ENTRY(setcr0)
580 movl 4(%esp), %eax
581 movl %eax, %cr0
582 ret
583 SET_SIZE(setcr0)
584
585 /*
586 * "lock mov %cr0" is used on processors which indicate it is
587 * supported via CPUID. Normally the 32 bit TPR is accessed via
588 * the local APIC.
589 */
590 ENTRY(getcr8)
591 lock
592 movl %cr0, %eax
593 ret
594 SET_SIZE(getcr8)
595
596 ENTRY(setcr8)
597 movl 4(%esp), %eax
598 lock
599 movl %eax, %cr0
600 ret
601 SET_SIZE(setcr8)
602
603 ENTRY(getcr2)
604 #if defined(__xpv)
605 movl %gs:CPU_VCPU_INFO, %eax
606 movl VCPU_INFO_ARCH_CR2(%eax), %eax
607 #else
608 movl %cr2, %eax
609 #endif
610 ret
611 SET_SIZE(getcr2)
612
613 ENTRY(getcr3)
614 movl %cr3, %eax
615 ret
616 SET_SIZE(getcr3)
617
618 #if !defined(__xpv)
619
620 ENTRY(setcr3)
621 movl 4(%esp), %eax
622 movl %eax, %cr3
623 ret
624 SET_SIZE(setcr3)
625
626 ENTRY(reload_cr3)
627 movl %cr3, %eax
628 movl %eax, %cr3
629 ret
630 SET_SIZE(reload_cr3)
631
632 #endif /* __xpv */
633
634 ENTRY(getcr4)
635 movl %cr4, %eax
636 ret
637 SET_SIZE(getcr4)
638
639 ENTRY(setcr4)
640 movl 4(%esp), %eax
641 movl %eax, %cr4
642 ret
643 SET_SIZE(setcr4)
644
645 #endif /* __i386 */
646 #endif /* __lint */
647
648 #if defined(__lint)
649
650 /*ARGSUSED*/
651 uint32_t
652 __cpuid_insn(struct cpuid_regs *regs)
653 { return (0); }
654
655 #else /* __lint */
656
657 #if defined(__amd64)
658
659 ENTRY(__cpuid_insn)
660 movq %rbx, %r8
661 movq %rcx, %r9
662 movq %rdx, %r11
663 movl (%rdi), %eax /* %eax = regs->cp_eax */
664 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
665 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
666 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
667 cpuid
668 movl %eax, (%rdi) /* regs->cp_eax = %eax */
669 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
670 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
671 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
672 movq %r8, %rbx
673 movq %r9, %rcx
674 movq %r11, %rdx
675 ret
676 SET_SIZE(__cpuid_insn)
677
678 #elif defined(__i386)
679
680 ENTRY(__cpuid_insn)
681 pushl %ebp
682 movl 0x8(%esp), %ebp /* %ebp = regs */
683 pushl %ebx
684 pushl %ecx
685 pushl %edx
686 movl (%ebp), %eax /* %eax = regs->cp_eax */
687 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
688 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
689 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
690 cpuid
691 movl %eax, (%ebp) /* regs->cp_eax = %eax */
692 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
693 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
694 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
695 popl %edx
696 popl %ecx
697 popl %ebx
698 popl %ebp
699 ret
700 SET_SIZE(__cpuid_insn)
701
702 #endif /* __i386 */
703 #endif /* __lint */
704
705 #if defined(__lint)
706
707 /*ARGSUSED*/
708 void
709 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
710 {}
711
712 #else /* __lint */
713
714 #if defined(__amd64)
715
716 ENTRY_NP(i86_monitor)
717 pushq %rbp
718 movq %rsp, %rbp
719 movq %rdi, %rax /* addr */
720 movq %rsi, %rcx /* extensions */
721 /* rdx contains input arg3: hints */
722 clflush (%rax)
723 .byte 0x0f, 0x01, 0xc8 /* monitor */
724 leave
725 ret
726 SET_SIZE(i86_monitor)
727
728 #elif defined(__i386)
729
730 ENTRY_NP(i86_monitor)
731 pushl %ebp
732 movl %esp, %ebp
733 movl 0x8(%ebp),%eax /* addr */
734 movl 0xc(%ebp),%ecx /* extensions */
735 movl 0x10(%ebp),%edx /* hints */
736 clflush (%eax)
737 .byte 0x0f, 0x01, 0xc8 /* monitor */
738 leave
739 ret
740 SET_SIZE(i86_monitor)
741
742 #endif /* __i386 */
743 #endif /* __lint */
744
745 #if defined(__lint)
746
747 /*ARGSUSED*/
748 void
749 i86_mwait(uint32_t data, uint32_t extensions)
750 {}
751
752 #else /* __lint */
753
754 #if defined(__amd64)
755
756 ENTRY_NP(i86_mwait)
757 pushq %rbp
758 call x86_md_clear
759 movq %rsp, %rbp
760 movq %rdi, %rax /* data */
761 movq %rsi, %rcx /* extensions */
762 .byte 0x0f, 0x01, 0xc9 /* mwait */
763 leave
764 ret
765 SET_SIZE(i86_mwait)
766
767 #elif defined(__i386)
768
769 ENTRY_NP(i86_mwait)
770 pushl %ebp
771 movl %esp, %ebp
772 movl 0x8(%ebp),%eax /* data */
773 movl 0xc(%ebp),%ecx /* extensions */
774 .byte 0x0f, 0x01, 0xc9 /* mwait */
775 leave
776 ret
777 SET_SIZE(i86_mwait)
778
779 #endif /* __i386 */
780 #endif /* __lint */
781
782 #if defined(__xpv)
783 /*
784 * Defined in C
785 */
786 #else
787
788 #if defined(__lint)
789
790 hrtime_t
791 tsc_read(void)
792 {
793 return (0);
794 }
795
796 #else /* __lint */
797
798 #if defined(__amd64)
799
800 ENTRY_NP(tsc_read)
801 movq %rbx, %r11
802 movl $0, %eax
803 cpuid
804 rdtsc
805 movq %r11, %rbx
806 shlq $32, %rdx
807 orq %rdx, %rax
808 ret
809 .globl _tsc_mfence_start
810 _tsc_mfence_start:
811 mfence
812 rdtsc
813 shlq $32, %rdx
814 orq %rdx, %rax
815 ret
816 .globl _tsc_mfence_end
817 _tsc_mfence_end:
818 .globl _tscp_start
819 _tscp_start:
824 .globl _tscp_end
825 _tscp_end:
826 .globl _no_rdtsc_start
827 _no_rdtsc_start:
828 xorl %edx, %edx
829 xorl %eax, %eax
830 ret
831 .globl _no_rdtsc_end
832 _no_rdtsc_end:
833 .globl _tsc_lfence_start
834 _tsc_lfence_start:
835 lfence
836 rdtsc
837 shlq $32, %rdx
838 orq %rdx, %rax
839 ret
840 .globl _tsc_lfence_end
841 _tsc_lfence_end:
842 SET_SIZE(tsc_read)
843
844 #else /* __i386 */
845
846 ENTRY_NP(tsc_read)
847 pushl %ebx
848 movl $0, %eax
849 cpuid
850 rdtsc
851 popl %ebx
852 ret
853 .globl _tsc_mfence_start
854 _tsc_mfence_start:
855 mfence
856 rdtsc
857 ret
858 .globl _tsc_mfence_end
859 _tsc_mfence_end:
860 .globl _tscp_start
861 _tscp_start:
862 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
863 ret
864 .globl _tscp_end
865 _tscp_end:
866 .globl _no_rdtsc_start
867 _no_rdtsc_start:
868 xorl %edx, %edx
869 xorl %eax, %eax
870 ret
871 .globl _no_rdtsc_end
872 _no_rdtsc_end:
873 .globl _tsc_lfence_start
874 _tsc_lfence_start:
875 lfence
876 rdtsc
877 ret
878 .globl _tsc_lfence_end
879 _tsc_lfence_end:
880 SET_SIZE(tsc_read)
881
882 #endif /* __i386 */
883
884 #endif /* __lint */
885
886
887 #endif /* __xpv */
888
889 #ifdef __lint
890 /*
891 * Do not use this function for obtaining clock tick. This
892 * is called by callers who do not need to have a guarenteed
893 * correct tick value. The proper routine to use is tsc_read().
894 */
895 u_longlong_t
896 randtick(void)
897 {
898 return (0);
899 }
900 #else
901 #if defined(__amd64)
902 ENTRY_NP(randtick)
903 rdtsc
904 shlq $32, %rdx
905 orq %rdx, %rax
906 ret
907 SET_SIZE(randtick)
908 #else
909 ENTRY_NP(randtick)
910 rdtsc
911 ret
912 SET_SIZE(randtick)
913 #endif /* __i386 */
914 #endif /* __lint */
915 /*
916 * Insert entryp after predp in a doubly linked list.
917 */
918
919 #if defined(__lint)
920
921 /*ARGSUSED*/
922 void
923 _insque(caddr_t entryp, caddr_t predp)
924 {}
925
926 #else /* __lint */
927
928 #if defined(__amd64)
929
930 ENTRY(_insque)
931 movq (%rsi), %rax /* predp->forw */
932 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
933 movq %rax, (%rdi) /* entryp->forw = predp->forw */
934 movq %rdi, (%rsi) /* predp->forw = entryp */
935 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
936 ret
937 SET_SIZE(_insque)
938
939 #elif defined(__i386)
940
941 ENTRY(_insque)
942 movl 8(%esp), %edx
943 movl 4(%esp), %ecx
944 movl (%edx), %eax /* predp->forw */
945 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
946 movl %eax, (%ecx) /* entryp->forw = predp->forw */
947 movl %ecx, (%edx) /* predp->forw = entryp */
948 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
949 ret
950 SET_SIZE(_insque)
951
952 #endif /* __i386 */
953 #endif /* __lint */
954
955 /*
956 * Remove entryp from a doubly linked list
957 */
958
959 #if defined(__lint)
960
961 /*ARGSUSED*/
962 void
963 _remque(caddr_t entryp)
964 {}
965
966 #else /* __lint */
967
968 #if defined(__amd64)
969
970 ENTRY(_remque)
971 movq (%rdi), %rax /* entry->forw */
972 movq CPTRSIZE(%rdi), %rdx /* entry->back */
973 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
974 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
975 ret
976 SET_SIZE(_remque)
977
978 #elif defined(__i386)
979
980 ENTRY(_remque)
981 movl 4(%esp), %ecx
982 movl (%ecx), %eax /* entry->forw */
983 movl CPTRSIZE(%ecx), %edx /* entry->back */
984 movl %eax, (%edx) /* entry->back->forw = entry->forw */
985 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
986 ret
987 SET_SIZE(_remque)
988
989 #endif /* __i386 */
990 #endif /* __lint */
991
992 /*
993 * Returns the number of
994 * non-NULL bytes in string argument.
995 */
996
997 #if defined(__lint)
998
999 /* ARGSUSED */
1000 size_t
1001 strlen(const char *str)
1002 { return (0); }
1003
1004 #else /* __lint */
1005
1006 #if defined(__amd64)
1007
1008 /*
1009 * This is close to a simple transliteration of a C version of this
1010 * routine. We should either just -make- this be a C version, or
1011 * justify having it in assembler by making it significantly faster.
1012 *
1013 * size_t
1014 * strlen(const char *s)
1015 * {
1016 * const char *s0;
1017 * #if defined(DEBUG)
1018 * if ((uintptr_t)s < KERNELBASE)
1019 * panic(.str_panic_msg);
1020 * #endif
1021 * for (s0 = s; *s; s++)
1022 * ;
1023 * return (s - s0);
1024 * }
1025 */
1026
1027 ENTRY(strlen)
1033 movq %rsp, %rbp
1034 leaq .str_panic_msg(%rip), %rdi
1035 xorl %eax, %eax
1036 call panic
1037 #endif /* DEBUG */
1038 str_valid:
1039 cmpb $0, (%rdi)
1040 movq %rdi, %rax
1041 je .null_found
1042 .align 4
1043 .strlen_loop:
1044 incq %rdi
1045 cmpb $0, (%rdi)
1046 jne .strlen_loop
1047 .null_found:
1048 subq %rax, %rdi
1049 movq %rdi, %rax
1050 ret
1051 SET_SIZE(strlen)
1052
1053 #elif defined(__i386)
1054
1055 ENTRY(strlen)
1056 #ifdef DEBUG
1057 movl postbootkernelbase, %eax
1058 cmpl %eax, 4(%esp)
1059 jae str_valid
1060 pushl %ebp
1061 movl %esp, %ebp
1062 pushl $.str_panic_msg
1063 call panic
1064 #endif /* DEBUG */
1065
1066 str_valid:
1067 movl 4(%esp), %eax /* %eax = string address */
1068 testl $3, %eax /* if %eax not word aligned */
1069 jnz .not_word_aligned /* goto .not_word_aligned */
1070 .align 4
1071 .word_aligned:
1072 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1073 movl $0x7f7f7f7f, %ecx
1074 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1075 addl $4, %eax /* next word */
1076 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1077 orl %edx, %ecx /* %ecx |= %edx */
1078 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1079 cmpl $0x80808080, %ecx /* if no null byte in this word */
1080 je .word_aligned /* goto .word_aligned */
1081 subl $4, %eax /* post-incremented */
1082 .not_word_aligned:
1083 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1084 je .null_found /* goto .null_found */
1085 incl %eax /* next byte */
1086 testl $3, %eax /* if %eax not word aligned */
1087 jnz .not_word_aligned /* goto .not_word_aligned */
1088 jmp .word_aligned /* goto .word_aligned */
1089 .align 4
1090 .null_found:
1091 subl 4(%esp), %eax /* %eax -= string address */
1092 ret
1093 SET_SIZE(strlen)
1094
1095 #endif /* __i386 */
1096
1097 #ifdef DEBUG
1098 .text
1099 .str_panic_msg:
1100 .string "strlen: argument below kernelbase"
1101 #endif /* DEBUG */
1102
1103 #endif /* __lint */
1104
1105 /*
1106 * Berkeley 4.3 introduced symbolically named interrupt levels
1107 * as a way deal with priority in a machine independent fashion.
1108 * Numbered priorities are machine specific, and should be
1109 * discouraged where possible.
1110 *
1111 * Note, for the machine specific priorities there are
1112 * examples listed for devices that use a particular priority.
1113 * It should not be construed that all devices of that
1114 * type should be at that priority. It is currently were
1115 * the current devices fit into the priority scheme based
1116 * upon time criticalness.
1117 *
1118 * The underlying assumption of these assignments is that
1119 * IPL 10 is the highest level from which a device
1120 * routine can call wakeup. Devices that interrupt from higher
1121 * levels are restricted in what they can do. If they need
1122 * kernels services they should schedule a routine at a lower
1123 * level (via software interrupt) to do the required
1124 * processing.
1125 *
1126 * Examples of this higher usage:
1127 * Level Usage
1128 * 14 Profiling clock (and PROM uart polling clock)
1129 * 12 Serial ports
1130 *
1131 * The serial ports request lower level processing on level 6.
1132 *
1133 * Also, almost all splN routines (where N is a number or a
1134 * mnemonic) will do a RAISE(), on the assumption that they are
1135 * never used to lower our priority.
1136 * The exceptions are:
1137 * spl8() Because you can't be above 15 to begin with!
1138 * splzs() Because this is used at boot time to lower our
1139 * priority, to allow the PROM to poll the uart.
1140 * spl0() Used to lower priority to 0.
1141 */
1142
1143 #if defined(__lint)
1144
1145 int spl0(void) { return (0); }
1146 int spl6(void) { return (0); }
1147 int spl7(void) { return (0); }
1148 int spl8(void) { return (0); }
1149 int splhigh(void) { return (0); }
1150 int splhi(void) { return (0); }
1151 int splzs(void) { return (0); }
1152
1153 /* ARGSUSED */
1154 void
1155 splx(int level)
1156 {}
1157
1158 #else /* __lint */
1159
1160 #if defined(__amd64)
1161
1162 #define SETPRI(level) \
1163 movl $/**/level, %edi; /* new priority */ \
1164 jmp do_splx /* redirect to do_splx */
1165
1166 #define RAISE(level) \
1167 movl $/**/level, %edi; /* new priority */ \
1168 jmp splr /* redirect to splr */
1169
1170 #elif defined(__i386)
1171
1172 #define SETPRI(level) \
1173 pushl $/**/level; /* new priority */ \
1174 call do_splx; /* invoke common splx code */ \
1175 addl $4, %esp; /* unstack arg */ \
1176 ret
1177
1178 #define RAISE(level) \
1179 pushl $/**/level; /* new priority */ \
1180 call splr; /* invoke common splr code */ \
1181 addl $4, %esp; /* unstack args */ \
1182 ret
1183
1184 #endif /* __i386 */
1185
1186 /* locks out all interrupts, including memory errors */
1187 ENTRY(spl8)
1188 SETPRI(15)
1189 SET_SIZE(spl8)
1190
1191 /* just below the level that profiling runs */
1192 ENTRY(spl7)
1193 RAISE(13)
1194 SET_SIZE(spl7)
1195
1196 /* sun specific - highest priority onboard serial i/o asy ports */
1197 ENTRY(splzs)
1198 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1199 SET_SIZE(splzs)
1200
1201 ENTRY(splhi)
1202 ALTENTRY(splhigh)
1203 ALTENTRY(spl6)
1204 ALTENTRY(i_ddi_splhigh)
1205
1206 RAISE(DISP_LEVEL)
1207
1208 SET_SIZE(i_ddi_splhigh)
1209 SET_SIZE(spl6)
1210 SET_SIZE(splhigh)
1211 SET_SIZE(splhi)
1212
1213 /* allow all interrupts */
1214 ENTRY(spl0)
1215 SETPRI(0)
1216 SET_SIZE(spl0)
1217
1218
1219 /* splx implementation */
1220 ENTRY(splx)
1221 jmp do_splx /* redirect to common splx code */
1222 SET_SIZE(splx)
1223
1224 #endif /* __lint */
1225
1226 #if defined(__i386)
1227
1228 /*
1229 * Read and write the %gs register
1230 */
1231
1232 #if defined(__lint)
1233
1234 /*ARGSUSED*/
1235 uint16_t
1236 getgs(void)
1237 { return (0); }
1238
1239 /*ARGSUSED*/
1240 void
1241 setgs(uint16_t sel)
1242 {}
1243
1244 #else /* __lint */
1245
1246 ENTRY(getgs)
1247 clr %eax
1248 movw %gs, %ax
1249 ret
1250 SET_SIZE(getgs)
1251
1252 ENTRY(setgs)
1253 movw 4(%esp), %gs
1254 ret
1255 SET_SIZE(setgs)
1256
1257 #endif /* __lint */
1258 #endif /* __i386 */
1259
1260 #if defined(__lint)
1261
1262 void
1263 pc_reset(void)
1264 {}
1265
1266 void
1267 efi_reset(void)
1268 {}
1269
1270 #else /* __lint */
1271
1272 ENTRY(wait_500ms)
1273 #if defined(__amd64)
1274 pushq %rbx
1275 #elif defined(__i386)
1276 push %ebx
1277 #endif
1278 movl $50000, %ebx
1279 1:
1280 call tenmicrosec
1281 decl %ebx
1282 jnz 1b
1283 #if defined(__amd64)
1284 popq %rbx
1285 #elif defined(__i386)
1286 pop %ebx
1287 #endif
1288 ret
1289 SET_SIZE(wait_500ms)
1290
1291 #define RESET_METHOD_KBC 1
1292 #define RESET_METHOD_PORT92 2
1293 #define RESET_METHOD_PCI 4
1294
1295 DGDEF3(pc_reset_methods, 4, 8)
1296 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1297
1298 ENTRY(pc_reset)
1299
1300 #if defined(__i386)
1301 testl $RESET_METHOD_KBC, pc_reset_methods
1302 #elif defined(__amd64)
1303 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1304 #endif
1305 jz 1f
1306
1307 /
1308 / Try the classic keyboard controller-triggered reset.
1309 /
1310 movw $0x64, %dx
1311 movb $0xfe, %al
1312 outb (%dx)
1313
1314 / Wait up to 500 milliseconds here for the keyboard controller
1315 / to pull the reset line. On some systems where the keyboard
1316 / controller is slow to pull the reset line, the next reset method
1317 / may be executed (which may be bad if those systems hang when the
1318 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1319 / and Ferrari 4000 (doesn't like the cf9 reset method))
1320
1321 call wait_500ms
1322
1323 1:
1324 #if defined(__i386)
1325 testl $RESET_METHOD_PORT92, pc_reset_methods
1326 #elif defined(__amd64)
1327 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1328 #endif
1329 jz 3f
1330
1331 /
1332 / Try port 0x92 fast reset
1333 /
1334 movw $0x92, %dx
1335 inb (%dx)
1336 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1337 je 1f
1338 testb $1, %al / If bit 0
1339 jz 2f / is clear, jump to perform the reset
1340 andb $0xfe, %al / otherwise,
1341 outb (%dx) / clear bit 0 first, then
1342 2:
1343 orb $1, %al / Set bit 0
1344 outb (%dx) / and reset the system
1345 1:
1346
1347 call wait_500ms
1348
1349 3:
1350 #if defined(__i386)
1351 testl $RESET_METHOD_PCI, pc_reset_methods
1352 #elif defined(__amd64)
1353 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1354 #endif
1355 jz 4f
1356
1357 / Try the PCI (soft) reset vector (should work on all modern systems,
1358 / but has been shown to cause problems on 450NX systems, and some newer
1359 / systems (e.g. ATI IXP400-equipped systems))
1360 / When resetting via this method, 2 writes are required. The first
1361 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1362 / power cycle).
1363 / The reset occurs on the second write, during bit 2's transition from
1364 / 0->1.
1365 movw $0xcf9, %dx
1366 movb $0x2, %al / Reset mode = hard, no power cycle
1367 outb (%dx)
1368 movb $0x6, %al
1369 outb (%dx)
1370
1371 call wait_500ms
1372
1373 4:
1374 /
1375 / port 0xcf9 failed also. Last-ditch effort is to
1376 / triple-fault the CPU.
1377 / Also, use triple fault for EFI firmware
1378 /
1379 ENTRY(efi_reset)
1380 #if defined(__amd64)
1381 pushq $0x0
1382 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1383 lidt (%rsp)
1384 #elif defined(__i386)
1385 pushl $0x0
1386 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1387 lidt (%esp)
1388 #endif
1389 int $0x0 / Trigger interrupt, generate triple-fault
1390
1391 cli
1392 hlt / Wait forever
1393 /*NOTREACHED*/
1394 SET_SIZE(efi_reset)
1395 SET_SIZE(pc_reset)
1396
1397 #endif /* __lint */
1398
1399 /*
1400 * C callable in and out routines
1401 */
1402
1403 #if defined(__lint)
1404
1405 /* ARGSUSED */
1406 void
1407 outl(int port_address, uint32_t val)
1408 {}
1409
1410 #else /* __lint */
1411
1412 #if defined(__amd64)
1413
1414 ENTRY(outl)
1415 movw %di, %dx
1416 movl %esi, %eax
1417 outl (%dx)
1418 ret
1419 SET_SIZE(outl)
1420
1421 #elif defined(__i386)
1422
1423 .set PORT, 4
1424 .set VAL, 8
1425
1426 ENTRY(outl)
1427 movw PORT(%esp), %dx
1428 movl VAL(%esp), %eax
1429 outl (%dx)
1430 ret
1431 SET_SIZE(outl)
1432
1433 #endif /* __i386 */
1434 #endif /* __lint */
1435
1436 #if defined(__lint)
1437
1438 /* ARGSUSED */
1439 void
1440 outw(int port_address, uint16_t val)
1441 {}
1442
1443 #else /* __lint */
1444
1445 #if defined(__amd64)
1446
1447 ENTRY(outw)
1448 movw %di, %dx
1449 movw %si, %ax
1450 D16 outl (%dx) /* XX64 why not outw? */
1451 ret
1452 SET_SIZE(outw)
1453
1454 #elif defined(__i386)
1455
1456 ENTRY(outw)
1457 movw PORT(%esp), %dx
1458 movw VAL(%esp), %ax
1459 D16 outl (%dx)
1460 ret
1461 SET_SIZE(outw)
1462
1463 #endif /* __i386 */
1464 #endif /* __lint */
1465
1466 #if defined(__lint)
1467
1468 /* ARGSUSED */
1469 void
1470 outb(int port_address, uint8_t val)
1471 {}
1472
1473 #else /* __lint */
1474
1475 #if defined(__amd64)
1476
1477 ENTRY(outb)
1478 movw %di, %dx
1479 movb %sil, %al
1480 outb (%dx)
1481 ret
1482 SET_SIZE(outb)
1483
1484 #elif defined(__i386)
1485
1486 ENTRY(outb)
1487 movw PORT(%esp), %dx
1488 movb VAL(%esp), %al
1489 outb (%dx)
1490 ret
1491 SET_SIZE(outb)
1492
1493 #endif /* __i386 */
1494 #endif /* __lint */
1495
1496 #if defined(__lint)
1497
1498 /* ARGSUSED */
1499 uint32_t
1500 inl(int port_address)
1501 { return (0); }
1502
1503 #else /* __lint */
1504
1505 #if defined(__amd64)
1506
1507 ENTRY(inl)
1508 xorl %eax, %eax
1509 movw %di, %dx
1510 inl (%dx)
1511 ret
1512 SET_SIZE(inl)
1513
1514 #elif defined(__i386)
1515
1516 ENTRY(inl)
1517 movw PORT(%esp), %dx
1518 inl (%dx)
1519 ret
1520 SET_SIZE(inl)
1521
1522 #endif /* __i386 */
1523 #endif /* __lint */
1524
1525 #if defined(__lint)
1526
1527 /* ARGSUSED */
1528 uint16_t
1529 inw(int port_address)
1530 { return (0); }
1531
1532 #else /* __lint */
1533
1534 #if defined(__amd64)
1535
1536 ENTRY(inw)
1537 xorl %eax, %eax
1538 movw %di, %dx
1539 D16 inl (%dx)
1540 ret
1541 SET_SIZE(inw)
1542
1543 #elif defined(__i386)
1544
1545 ENTRY(inw)
1546 subl %eax, %eax
1547 movw PORT(%esp), %dx
1548 D16 inl (%dx)
1549 ret
1550 SET_SIZE(inw)
1551
1552 #endif /* __i386 */
1553 #endif /* __lint */
1554
1555
1556 #if defined(__lint)
1557
1558 /* ARGSUSED */
1559 uint8_t
1560 inb(int port_address)
1561 { return (0); }
1562
1563 #else /* __lint */
1564
1565 #if defined(__amd64)
1566
1567 ENTRY(inb)
1568 xorl %eax, %eax
1569 movw %di, %dx
1570 inb (%dx)
1571 ret
1572 SET_SIZE(inb)
1573
1574 #elif defined(__i386)
1575
1576 ENTRY(inb)
1577 subl %eax, %eax
1578 movw PORT(%esp), %dx
1579 inb (%dx)
1580 ret
1581 SET_SIZE(inb)
1582
1583 #endif /* __i386 */
1584 #endif /* __lint */
1585
1586
1587 #if defined(__lint)
1588
1589 /* ARGSUSED */
1590 void
1591 repoutsw(int port, uint16_t *addr, int cnt)
1592 {}
1593
1594 #else /* __lint */
1595
1596 #if defined(__amd64)
1597
1598 ENTRY(repoutsw)
1599 movl %edx, %ecx
1600 movw %di, %dx
1601 rep
1602 D16 outsl
1603 ret
1604 SET_SIZE(repoutsw)
1605
1606 #elif defined(__i386)
1607
1608 /*
1609 * The arguments and saved registers are on the stack in the
1610 * following order:
1611 * | cnt | +16
1612 * | *addr | +12
1613 * | port | +8
1614 * | eip | +4
1615 * | esi | <-- %esp
1616 * If additional values are pushed onto the stack, make sure
1617 * to adjust the following constants accordingly.
1618 */
1619 .set PORT, 8
1620 .set ADDR, 12
1621 .set COUNT, 16
1622
1623 ENTRY(repoutsw)
1624 pushl %esi
1625 movl PORT(%esp), %edx
1626 movl ADDR(%esp), %esi
1627 movl COUNT(%esp), %ecx
1628 rep
1629 D16 outsl
1630 popl %esi
1631 ret
1632 SET_SIZE(repoutsw)
1633
1634 #endif /* __i386 */
1635 #endif /* __lint */
1636
1637
1638 #if defined(__lint)
1639
1640 /* ARGSUSED */
1641 void
1642 repinsw(int port_addr, uint16_t *addr, int cnt)
1643 {}
1644
1645 #else /* __lint */
1646
1647 #if defined(__amd64)
1648
1649 ENTRY(repinsw)
1650 movl %edx, %ecx
1651 movw %di, %dx
1652 rep
1653 D16 insl
1654 ret
1655 SET_SIZE(repinsw)
1656
1657 #elif defined(__i386)
1658
1659 ENTRY(repinsw)
1660 pushl %edi
1661 movl PORT(%esp), %edx
1662 movl ADDR(%esp), %edi
1663 movl COUNT(%esp), %ecx
1664 rep
1665 D16 insl
1666 popl %edi
1667 ret
1668 SET_SIZE(repinsw)
1669
1670 #endif /* __i386 */
1671 #endif /* __lint */
1672
1673
1674 #if defined(__lint)
1675
1676 /* ARGSUSED */
1677 void
1678 repinsb(int port, uint8_t *addr, int count)
1679 {}
1680
1681 #else /* __lint */
1682
1683 #if defined(__amd64)
1684
1685 ENTRY(repinsb)
1686 movl %edx, %ecx
1687 movw %di, %dx
1688 movq %rsi, %rdi
1689 rep
1690 insb
1691 ret
1692 SET_SIZE(repinsb)
1693
1694 #elif defined(__i386)
1695
1696 /*
1697 * The arguments and saved registers are on the stack in the
1698 * following order:
1699 * | cnt | +16
1700 * | *addr | +12
1701 * | port | +8
1702 * | eip | +4
1703 * | esi | <-- %esp
1704 * If additional values are pushed onto the stack, make sure
1705 * to adjust the following constants accordingly.
1706 */
1707 .set IO_PORT, 8
1708 .set IO_ADDR, 12
1709 .set IO_COUNT, 16
1710
1711 ENTRY(repinsb)
1712 pushl %edi
1713 movl IO_ADDR(%esp), %edi
1714 movl IO_COUNT(%esp), %ecx
1715 movl IO_PORT(%esp), %edx
1716 rep
1717 insb
1718 popl %edi
1719 ret
1720 SET_SIZE(repinsb)
1721
1722 #endif /* __i386 */
1723 #endif /* __lint */
1724
1725
1726 /*
1727 * Input a stream of 32-bit words.
1728 * NOTE: count is a DWORD count.
1729 */
1730 #if defined(__lint)
1731
1732 /* ARGSUSED */
1733 void
1734 repinsd(int port, uint32_t *addr, int count)
1735 {}
1736
1737 #else /* __lint */
1738
1739 #if defined(__amd64)
1740
1741 ENTRY(repinsd)
1742 movl %edx, %ecx
1743 movw %di, %dx
1744 movq %rsi, %rdi
1745 rep
1746 insl
1747 ret
1748 SET_SIZE(repinsd)
1749
1750 #elif defined(__i386)
1751
1752 ENTRY(repinsd)
1753 pushl %edi
1754 movl IO_ADDR(%esp), %edi
1755 movl IO_COUNT(%esp), %ecx
1756 movl IO_PORT(%esp), %edx
1757 rep
1758 insl
1759 popl %edi
1760 ret
1761 SET_SIZE(repinsd)
1762
1763 #endif /* __i386 */
1764 #endif /* __lint */
1765
1766 /*
1767 * Output a stream of bytes
1768 * NOTE: count is a byte count
1769 */
1770 #if defined(__lint)
1771
1772 /* ARGSUSED */
1773 void
1774 repoutsb(int port, uint8_t *addr, int count)
1775 {}
1776
1777 #else /* __lint */
1778
1779 #if defined(__amd64)
1780
1781 ENTRY(repoutsb)
1782 movl %edx, %ecx
1783 movw %di, %dx
1784 rep
1785 outsb
1786 ret
1787 SET_SIZE(repoutsb)
1788
1789 #elif defined(__i386)
1790
1791 ENTRY(repoutsb)
1792 pushl %esi
1793 movl IO_ADDR(%esp), %esi
1794 movl IO_COUNT(%esp), %ecx
1795 movl IO_PORT(%esp), %edx
1796 rep
1797 outsb
1798 popl %esi
1799 ret
1800 SET_SIZE(repoutsb)
1801
1802 #endif /* __i386 */
1803 #endif /* __lint */
1804
1805 /*
1806 * Output a stream of 32-bit words
1807 * NOTE: count is a DWORD count
1808 */
1809 #if defined(__lint)
1810
1811 /* ARGSUSED */
1812 void
1813 repoutsd(int port, uint32_t *addr, int count)
1814 {}
1815
1816 #else /* __lint */
1817
1818 #if defined(__amd64)
1819
1820 ENTRY(repoutsd)
1821 movl %edx, %ecx
1822 movw %di, %dx
1823 rep
1824 outsl
1825 ret
1826 SET_SIZE(repoutsd)
1827
1828 #elif defined(__i386)
1829
1830 ENTRY(repoutsd)
1831 pushl %esi
1832 movl IO_ADDR(%esp), %esi
1833 movl IO_COUNT(%esp), %ecx
1834 movl IO_PORT(%esp), %edx
1835 rep
1836 outsl
1837 popl %esi
1838 ret
1839 SET_SIZE(repoutsd)
1840
1841 #endif /* __i386 */
1842 #endif /* __lint */
1843
1844 /*
1845 * void int3(void)
1846 * void int18(void)
1847 * void int20(void)
1848 * void int_cmci(void)
1849 */
1850
1851 #if defined(__lint)
1852
1853 void
1854 int3(void)
1855 {}
1856
1857 void
1858 int18(void)
1859 {}
1860
1861 void
1862 int20(void)
1863 {}
1864
1865 void
1866 int_cmci(void)
1867 {}
1868
1869 #else /* __lint */
1870
1871 ENTRY(int3)
1872 int $T_BPTFLT
1873 ret
1874 SET_SIZE(int3)
1875
1876 ENTRY(int18)
1877 int $T_MCE
1878 ret
1879 SET_SIZE(int18)
1880
1881 ENTRY(int20)
1882 movl boothowto, %eax
1883 andl $RB_DEBUG, %eax
1884 jz 1f
1885
1886 int $T_DBGENTR
1887 1:
1888 rep; ret /* use 2 byte return instruction when branch target */
1889 /* AMD Software Optimization Guide - Section 6.2 */
1890 SET_SIZE(int20)
1891
1892 ENTRY(int_cmci)
1893 int $T_ENOEXTFLT
1894 ret
1895 SET_SIZE(int_cmci)
1896
1897 #endif /* __lint */
1898
1899 #if defined(__lint)
1900
1901 /* ARGSUSED */
1902 int
1903 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1904 { return (0); }
1905
1906 #else /* __lint */
1907
1908 #if defined(__amd64)
1909
1910 ENTRY(scanc)
1911 /* rdi == size */
1912 /* rsi == cp */
1913 /* rdx == table */
1914 /* rcx == mask */
1915 addq %rsi, %rdi /* end = &cp[size] */
1916 .scanloop:
1917 cmpq %rdi, %rsi /* while (cp < end */
1918 jnb .scandone
1919 movzbq (%rsi), %r8 /* %r8 = *cp */
1920 incq %rsi /* cp++ */
1921 testb %cl, (%r8, %rdx)
1922 jz .scanloop /* && (table[*cp] & mask) == 0) */
1923 decq %rsi /* (fix post-increment) */
1924 .scandone:
1925 movl %edi, %eax
1926 subl %esi, %eax /* return (end - cp) */
1927 ret
1928 SET_SIZE(scanc)
1929
1930 #elif defined(__i386)
1931
1932 ENTRY(scanc)
1933 pushl %edi
1934 pushl %esi
1935 movb 24(%esp), %cl /* mask = %cl */
1936 movl 16(%esp), %esi /* cp = %esi */
1937 movl 20(%esp), %edx /* table = %edx */
1938 movl %esi, %edi
1939 addl 12(%esp), %edi /* end = &cp[size]; */
1940 .scanloop:
1941 cmpl %edi, %esi /* while (cp < end */
1942 jnb .scandone
1943 movzbl (%esi), %eax /* %al = *cp */
1944 incl %esi /* cp++ */
1945 movb (%edx, %eax), %al /* %al = table[*cp] */
1946 testb %al, %cl
1947 jz .scanloop /* && (table[*cp] & mask) == 0) */
1948 dec %esi /* post-incremented */
1949 .scandone:
1950 movl %edi, %eax
1951 subl %esi, %eax /* return (end - cp) */
1952 popl %esi
1953 popl %edi
1954 ret
1955 SET_SIZE(scanc)
1956
1957 #endif /* __i386 */
1958 #endif /* __lint */
1959
1960 /*
1961 * Replacement functions for ones that are normally inlined.
1962 * In addition to the copy in i86.il, they are defined here just in case.
1963 */
1964
1965 #if defined(__lint)
1966
1967 ulong_t
1968 intr_clear(void)
1969 { return (0); }
1970
1971 ulong_t
1972 clear_int_flag(void)
1973 { return (0); }
1974
1975 #else /* __lint */
1976
1977 #if defined(__amd64)
1978
1979 ENTRY(intr_clear)
1980 ENTRY(clear_int_flag)
1981 pushfq
1982 popq %rax
1983 #if defined(__xpv)
1984 leaq xpv_panicking, %rdi
1985 movl (%rdi), %edi
1986 cmpl $0, %edi
1987 jne 2f
1988 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1989 /*
1990 * Synthesize the PS_IE bit from the event mask bit
1991 */
1992 andq $_BITNOT(PS_IE), %rax
1993 testb $1, %dl
1994 jnz 1f
1995 orq $PS_IE, %rax
1996 1:
1997 ret
1998 2:
1999 #endif
2000 CLI(%rdi)
2001 ret
2002 SET_SIZE(clear_int_flag)
2003 SET_SIZE(intr_clear)
2004
2005 #elif defined(__i386)
2006
2007 ENTRY(intr_clear)
2008 ENTRY(clear_int_flag)
2009 pushfl
2010 popl %eax
2011 #if defined(__xpv)
2012 leal xpv_panicking, %edx
2013 movl (%edx), %edx
2014 cmpl $0, %edx
2015 jne 2f
2016 CLIRET(%edx, %cl) /* returns event mask in %cl */
2017 /*
2018 * Synthesize the PS_IE bit from the event mask bit
2019 */
2020 andl $_BITNOT(PS_IE), %eax
2021 testb $1, %cl
2022 jnz 1f
2023 orl $PS_IE, %eax
2024 1:
2025 ret
2026 2:
2027 #endif
2028 CLI(%edx)
2029 ret
2030 SET_SIZE(clear_int_flag)
2031 SET_SIZE(intr_clear)
2032
2033 #endif /* __i386 */
2034 #endif /* __lint */
2035
2036 #if defined(__lint)
2037
2038 struct cpu *
2039 curcpup(void)
2040 { return 0; }
2041
2042 #else /* __lint */
2043
2044 #if defined(__amd64)
2045
2046 ENTRY(curcpup)
2047 movq %gs:CPU_SELF, %rax
2048 ret
2049 SET_SIZE(curcpup)
2050
2051 #elif defined(__i386)
2052
2053 ENTRY(curcpup)
2054 movl %gs:CPU_SELF, %eax
2055 ret
2056 SET_SIZE(curcpup)
2057
2058 #endif /* __i386 */
2059 #endif /* __lint */
2060
2061 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2062 * These functions reverse the byte order of the input parameter and returns
2063 * the result. This is to convert the byte order from host byte order
2064 * (little endian) to network byte order (big endian), or vice versa.
2065 */
2066
2067 #if defined(__lint)
2068
2069 uint64_t
2070 htonll(uint64_t i)
2071 { return (i); }
2072
2073 uint64_t
2074 ntohll(uint64_t i)
2075 { return (i); }
2076
2077 uint32_t
2078 htonl(uint32_t i)
2079 { return (i); }
2080
2081 uint32_t
2082 ntohl(uint32_t i)
2083 { return (i); }
2084
2085 uint16_t
2086 htons(uint16_t i)
2087 { return (i); }
2088
2089 uint16_t
2090 ntohs(uint16_t i)
2091 { return (i); }
2092
2093 #else /* __lint */
2094
2095 #if defined(__amd64)
2096
2097 ENTRY(htonll)
2098 ALTENTRY(ntohll)
2099 movq %rdi, %rax
2100 bswapq %rax
2101 ret
2102 SET_SIZE(ntohll)
2103 SET_SIZE(htonll)
2104
2105 /* XX64 there must be shorter sequences for this */
2106 ENTRY(htonl)
2107 ALTENTRY(ntohl)
2108 movl %edi, %eax
2109 bswap %eax
2110 ret
2111 SET_SIZE(ntohl)
2112 SET_SIZE(htonl)
2113
2114 /* XX64 there must be better sequences for this */
2115 ENTRY(htons)
2116 ALTENTRY(ntohs)
2117 movl %edi, %eax
2118 bswap %eax
2119 shrl $16, %eax
2120 ret
2121 SET_SIZE(ntohs)
2122 SET_SIZE(htons)
2123
2124 #elif defined(__i386)
2125
2126 ENTRY(htonll)
2127 ALTENTRY(ntohll)
2128 movl 4(%esp), %edx
2129 movl 8(%esp), %eax
2130 bswap %edx
2131 bswap %eax
2132 ret
2133 SET_SIZE(ntohll)
2134 SET_SIZE(htonll)
2135
2136 ENTRY(htonl)
2137 ALTENTRY(ntohl)
2138 movl 4(%esp), %eax
2139 bswap %eax
2140 ret
2141 SET_SIZE(ntohl)
2142 SET_SIZE(htonl)
2143
2144 ENTRY(htons)
2145 ALTENTRY(ntohs)
2146 movl 4(%esp), %eax
2147 bswap %eax
2148 shrl $16, %eax
2149 ret
2150 SET_SIZE(ntohs)
2151 SET_SIZE(htons)
2152
2153 #endif /* __i386 */
2154 #endif /* __lint */
2155
2156
2157 #if defined(__lint)
2158
2159 /* ARGSUSED */
2160 void
2161 intr_restore(ulong_t i)
2162 { return; }
2163
2164 /* ARGSUSED */
2165 void
2166 restore_int_flag(ulong_t i)
2167 { return; }
2168
2169 #else /* __lint */
2170
2171 #if defined(__amd64)
2172
2173 ENTRY(intr_restore)
2174 ENTRY(restore_int_flag)
2175 testq $PS_IE, %rdi
2176 jz 1f
2177 #if defined(__xpv)
2178 leaq xpv_panicking, %rsi
2179 movl (%rsi), %esi
2180 cmpl $0, %esi
2181 jne 1f
2182 /*
2183 * Since we're -really- running unprivileged, our attempt
2184 * to change the state of the IF bit will be ignored.
2185 * The virtual IF bit is tweaked by CLI and STI.
2186 */
2187 IE_TO_EVENT_MASK(%rsi, %rdi)
2188 #else
2189 sti
2190 #endif
2191 1:
2192 ret
2193 SET_SIZE(restore_int_flag)
2194 SET_SIZE(intr_restore)
2195
2196 #elif defined(__i386)
2197
2198 ENTRY(intr_restore)
2199 ENTRY(restore_int_flag)
2200 testl $PS_IE, 4(%esp)
2201 jz 1f
2202 #if defined(__xpv)
2203 leal xpv_panicking, %edx
2204 movl (%edx), %edx
2205 cmpl $0, %edx
2206 jne 1f
2207 /*
2208 * Since we're -really- running unprivileged, our attempt
2209 * to change the state of the IF bit will be ignored.
2210 * The virtual IF bit is tweaked by CLI and STI.
2211 */
2212 IE_TO_EVENT_MASK(%edx, 4(%esp))
2213 #else
2214 sti
2215 #endif
2216 1:
2217 ret
2218 SET_SIZE(restore_int_flag)
2219 SET_SIZE(intr_restore)
2220
2221 #endif /* __i386 */
2222 #endif /* __lint */
2223
2224 #if defined(__lint)
2225
2226 void
2227 sti(void)
2228 {}
2229
2230 void
2231 cli(void)
2232 {}
2233
2234 #else /* __lint */
2235
2236 ENTRY(sti)
2237 STI
2238 ret
2239 SET_SIZE(sti)
2240
2241 ENTRY(cli)
2242 #if defined(__amd64)
2243 CLI(%rax)
2244 #elif defined(__i386)
2245 CLI(%eax)
2246 #endif /* __i386 */
2247 ret
2248 SET_SIZE(cli)
2249
2250 #endif /* __lint */
2251
2252 #if defined(__lint)
2253
2254 dtrace_icookie_t
2255 dtrace_interrupt_disable(void)
2256 { return (0); }
2257
2258 #else /* __lint */
2259
2260 #if defined(__amd64)
2261
2262 ENTRY(dtrace_interrupt_disable)
2263 pushfq
2264 popq %rax
2265 #if defined(__xpv)
2266 leaq xpv_panicking, %rdi
2267 movl (%rdi), %edi
2268 cmpl $0, %edi
2269 jne .dtrace_interrupt_disable_done
2270 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2271 /*
2272 * Synthesize the PS_IE bit from the event mask bit
2273 */
2274 andq $_BITNOT(PS_IE), %rax
2275 testb $1, %dl
2276 jnz .dtrace_interrupt_disable_done
2277 orq $PS_IE, %rax
2278 #else
2279 CLI(%rdx)
2280 #endif
2281 .dtrace_interrupt_disable_done:
2282 ret
2283 SET_SIZE(dtrace_interrupt_disable)
2284
2285 #elif defined(__i386)
2286
2287 ENTRY(dtrace_interrupt_disable)
2288 pushfl
2289 popl %eax
2290 #if defined(__xpv)
2291 leal xpv_panicking, %edx
2292 movl (%edx), %edx
2293 cmpl $0, %edx
2294 jne .dtrace_interrupt_disable_done
2295 CLIRET(%edx, %cl) /* returns event mask in %cl */
2296 /*
2297 * Synthesize the PS_IE bit from the event mask bit
2298 */
2299 andl $_BITNOT(PS_IE), %eax
2300 testb $1, %cl
2301 jnz .dtrace_interrupt_disable_done
2302 orl $PS_IE, %eax
2303 #else
2304 CLI(%edx)
2305 #endif
2306 .dtrace_interrupt_disable_done:
2307 ret
2308 SET_SIZE(dtrace_interrupt_disable)
2309
2310 #endif /* __i386 */
2311 #endif /* __lint */
2312
2313 #if defined(__lint)
2314
2315 /*ARGSUSED*/
2316 void
2317 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2318 {}
2319
2320 #else /* __lint */
2321
2322 #if defined(__amd64)
2323
2324 ENTRY(dtrace_interrupt_enable)
2325 pushq %rdi
2326 popfq
2327 #if defined(__xpv)
2328 leaq xpv_panicking, %rdx
2329 movl (%rdx), %edx
2330 cmpl $0, %edx
2331 jne .dtrace_interrupt_enable_done
2332 /*
2333 * Since we're -really- running unprivileged, our attempt
2334 * to change the state of the IF bit will be ignored. The
2335 * virtual IF bit is tweaked by CLI and STI.
2336 */
2337 IE_TO_EVENT_MASK(%rdx, %rdi)
2338 #endif
2339 .dtrace_interrupt_enable_done:
2340 ret
2341 SET_SIZE(dtrace_interrupt_enable)
2342
2343 #elif defined(__i386)
2344
2345 ENTRY(dtrace_interrupt_enable)
2346 movl 4(%esp), %eax
2347 pushl %eax
2348 popfl
2349 #if defined(__xpv)
2350 leal xpv_panicking, %edx
2351 movl (%edx), %edx
2352 cmpl $0, %edx
2353 jne .dtrace_interrupt_enable_done
2354 /*
2355 * Since we're -really- running unprivileged, our attempt
2356 * to change the state of the IF bit will be ignored. The
2357 * virtual IF bit is tweaked by CLI and STI.
2358 */
2359 IE_TO_EVENT_MASK(%edx, %eax)
2360 #endif
2361 .dtrace_interrupt_enable_done:
2362 ret
2363 SET_SIZE(dtrace_interrupt_enable)
2364
2365 #endif /* __i386 */
2366 #endif /* __lint */
2367
2368
2369 #if defined(lint)
2370
2371 void
2372 dtrace_membar_producer(void)
2373 {}
2374
2375 void
2376 dtrace_membar_consumer(void)
2377 {}
2378
2379 #else /* __lint */
2380
2381 ENTRY(dtrace_membar_producer)
2382 rep; ret /* use 2 byte return instruction when branch target */
2383 /* AMD Software Optimization Guide - Section 6.2 */
2384 SET_SIZE(dtrace_membar_producer)
2385
2386 ENTRY(dtrace_membar_consumer)
2387 rep; ret /* use 2 byte return instruction when branch target */
2388 /* AMD Software Optimization Guide - Section 6.2 */
2389 SET_SIZE(dtrace_membar_consumer)
2390
2391 #endif /* __lint */
2392
2393 #if defined(__lint)
2394
2395 kthread_id_t
2396 threadp(void)
2397 { return ((kthread_id_t)0); }
2398
2399 #else /* __lint */
2400
2401 #if defined(__amd64)
2402
2403 ENTRY(threadp)
2404 movq %gs:CPU_THREAD, %rax
2405 ret
2406 SET_SIZE(threadp)
2407
2408 #elif defined(__i386)
2409
2410 ENTRY(threadp)
2411 movl %gs:CPU_THREAD, %eax
2412 ret
2413 SET_SIZE(threadp)
2414
2415 #endif /* __i386 */
2416 #endif /* __lint */
2417
2418 /*
2419 * Checksum routine for Internet Protocol Headers
2420 */
2421
2422 #if defined(__lint)
2423
2424 /* ARGSUSED */
2425 unsigned int
2426 ip_ocsum(
2427 ushort_t *address, /* ptr to 1st message buffer */
2428 int halfword_count, /* length of data */
2429 unsigned int sum) /* partial checksum */
2430 {
2431 int i;
2432 unsigned int psum = 0; /* partial sum */
2433
2434 for (i = 0; i < halfword_count; i++, address++) {
2435 psum += *address;
2436 }
2437
2438 while ((psum >> 16) != 0) {
2439 psum = (psum & 0xffff) + (psum >> 16);
2440 }
2441
2442 psum += sum;
2443
2444 while ((psum >> 16) != 0) {
2445 psum = (psum & 0xffff) + (psum >> 16);
2446 }
2447
2448 return (psum);
2449 }
2450
2451 #else /* __lint */
2452
2453 #if defined(__amd64)
2454
2455 ENTRY(ip_ocsum)
2456 pushq %rbp
2457 movq %rsp, %rbp
2458 #ifdef DEBUG
2459 movq postbootkernelbase(%rip), %rax
2460 cmpq %rax, %rdi
2461 jnb 1f
2462 xorl %eax, %eax
2463 movq %rdi, %rsi
2464 leaq .ip_ocsum_panic_msg(%rip), %rdi
2465 call panic
2466 /*NOTREACHED*/
2467 .ip_ocsum_panic_msg:
2468 .string "ip_ocsum: address 0x%p below kernelbase\n"
2469 1:
2470 #endif
2471 movl %esi, %ecx /* halfword_count */
2472 movq %rdi, %rsi /* address */
2473 /* partial sum in %edx */
2474 xorl %eax, %eax
2553 .size_aligned:
2554 movl %ecx, %edi
2555 shrl $1, %ecx
2556 shl $1, %edi
2557 subq $64, %rdi
2558 addq %rdi, %rsi
2559 leaq .ip_ocsum_jmptbl(%rip), %rdi
2560 leaq (%rdi, %rcx, 8), %rdi
2561 xorl %ecx, %ecx
2562 clc
2563 movq (%rdi), %rdi
2564 INDIRECT_JMP_REG(rdi)
2565
2566 .align 8
2567 .ip_ocsum_jmptbl:
2568 .quad .only0, .only4, .only8, .only12, .only16, .only20
2569 .quad .only24, .only28, .only32, .only36, .only40, .only44
2570 .quad .only48, .only52, .only56, .only60
2571 SET_SIZE(ip_ocsum)
2572
2573 #elif defined(__i386)
2574
2575 ENTRY(ip_ocsum)
2576 pushl %ebp
2577 movl %esp, %ebp
2578 pushl %ebx
2579 pushl %esi
2580 pushl %edi
2581 movl 12(%ebp), %ecx /* count of half words */
2582 movl 16(%ebp), %edx /* partial checksum */
2583 movl 8(%ebp), %esi
2584 xorl %eax, %eax
2585 testl %ecx, %ecx
2586 jz .ip_ocsum_done
2587
2588 testl $3, %esi
2589 jnz .ip_csum_notaligned
2590 .ip_csum_aligned:
2591 .next_iter:
2592 subl $32, %ecx
2593 jl .less_than_32
2594
2595 addl 0(%esi), %edx
2596 .only60:
2597 adcl 4(%esi), %eax
2598 .only56:
2599 adcl 8(%esi), %edx
2600 .only52:
2601 adcl 12(%esi), %eax
2602 .only48:
2603 adcl 16(%esi), %edx
2604 .only44:
2605 adcl 20(%esi), %eax
2606 .only40:
2607 adcl 24(%esi), %edx
2608 .only36:
2609 adcl 28(%esi), %eax
2610 .only32:
2611 adcl 32(%esi), %edx
2612 .only28:
2613 adcl 36(%esi), %eax
2614 .only24:
2615 adcl 40(%esi), %edx
2616 .only20:
2617 adcl 44(%esi), %eax
2618 .only16:
2619 adcl 48(%esi), %edx
2620 .only12:
2621 adcl 52(%esi), %eax
2622 .only8:
2623 adcl 56(%esi), %edx
2624 .only4:
2625 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2626 .only0:
2627 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2628 adcl $0, %eax
2629
2630 addl $64, %esi
2631 andl %ecx, %ecx
2632 jnz .next_iter
2633
2634 .ip_ocsum_done:
2635 addl %eax, %edx
2636 adcl $0, %edx
2637 movl %edx, %eax /* form a 16 bit checksum by */
2638 shrl $16, %eax /* adding two halves of 32 bit checksum */
2639 addw %dx, %ax
2640 adcw $0, %ax
2641 andl $0xffff, %eax
2642 popl %edi /* restore registers */
2643 popl %esi
2644 popl %ebx
2645 leave
2646 ret
2647
2648 .ip_csum_notaligned:
2649 xorl %edi, %edi
2650 movw (%esi), %di
2651 addl %edi, %edx
2652 adcl $0, %edx
2653 addl $2, %esi
2654 decl %ecx
2655 jmp .ip_csum_aligned
2656
2657 .less_than_32:
2658 addl $32, %ecx
2659 testl $1, %ecx
2660 jz .size_aligned
2661 andl $0xfe, %ecx
2662 movzwl (%esi, %ecx, 2), %edi
2663 addl %edi, %edx
2664 adcl $0, %edx
2665 .size_aligned:
2666 movl %ecx, %edi
2667 shrl $1, %ecx
2668 shl $1, %edi
2669 subl $64, %edi
2670 addl %edi, %esi
2671 movl $.ip_ocsum_jmptbl, %edi
2672 lea (%edi, %ecx, 4), %edi
2673 xorl %ecx, %ecx
2674 clc
2675 jmp *(%edi)
2676 SET_SIZE(ip_ocsum)
2677
2678 .data
2679 .align 4
2680
2681 .ip_ocsum_jmptbl:
2682 .long .only0, .only4, .only8, .only12, .only16, .only20
2683 .long .only24, .only28, .only32, .only36, .only40, .only44
2684 .long .only48, .only52, .only56, .only60
2685
2686
2687 #endif /* __i386 */
2688 #endif /* __lint */
2689
2690 /*
2691 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2692 * Provided to manipulate hrtime_t values.
2693 */
2694 #if defined(__lint)
2695
2696 /* result = a * b; */
2697
2698 /* ARGSUSED */
2699 unsigned long long
2700 mul32(uint_t a, uint_t b)
2701 { return (0); }
2702
2703 #else /* __lint */
2704
2705 #if defined(__amd64)
2706
2707 ENTRY(mul32)
2708 xorl %edx, %edx /* XX64 joe, paranoia? */
2709 movl %edi, %eax
2710 mull %esi
2711 shlq $32, %rdx
2712 orq %rdx, %rax
2713 ret
2714 SET_SIZE(mul32)
2715
2716 #elif defined(__i386)
2717
2718 ENTRY(mul32)
2719 movl 8(%esp), %eax
2720 movl 4(%esp), %ecx
2721 mull %ecx
2722 ret
2723 SET_SIZE(mul32)
2724
2725 #endif /* __i386 */
2726 #endif /* __lint */
2727
2728 #if defined(notused)
2729 #if defined(__lint)
2730 /* ARGSUSED */
2731 void
2732 load_pte64(uint64_t *pte, uint64_t pte_value)
2733 {}
2734 #else /* __lint */
2735 .globl load_pte64
2736 load_pte64:
2737 movl 4(%esp), %eax
2738 movl 8(%esp), %ecx
2739 movl 12(%esp), %edx
2740 movl %edx, 4(%eax)
2741 movl %ecx, (%eax)
2742 ret
2743 #endif /* __lint */
2744 #endif /* notused */
2745
2746 #if defined(__lint)
2747
2748 /*ARGSUSED*/
2749 void
2750 scan_memory(caddr_t addr, size_t size)
2751 {}
2752
2753 #else /* __lint */
2754
2755 #if defined(__amd64)
2756
2757 ENTRY(scan_memory)
2758 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2759 jz .scanm_done
2760 movq %rsi, %rcx /* move count into rep control register */
2761 movq %rdi, %rsi /* move addr into lodsq control reg. */
2762 rep lodsq /* scan the memory range */
2763 .scanm_done:
2764 rep; ret /* use 2 byte return instruction when branch target */
2765 /* AMD Software Optimization Guide - Section 6.2 */
2766 SET_SIZE(scan_memory)
2767
2768 #elif defined(__i386)
2769
2770 ENTRY(scan_memory)
2771 pushl %ecx
2772 pushl %esi
2773 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2774 shrl $2, %ecx /* convert from byte count to word count */
2775 jz .scanm_done
2776 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2777 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2778 lodsl
2779 .scanm_done:
2780 popl %esi
2781 popl %ecx
2782 ret
2783 SET_SIZE(scan_memory)
2784
2785 #endif /* __i386 */
2786 #endif /* __lint */
2787
2788
2789 #if defined(__lint)
2790
2791 /*ARGSUSED */
2792 int
2793 lowbit(ulong_t i)
2794 { return (0); }
2795
2796 #else /* __lint */
2797
2798 #if defined(__amd64)
2799
2800 ENTRY(lowbit)
2801 movl $-1, %eax
2802 bsfq %rdi, %rdi
2803 cmovnz %edi, %eax
2804 incl %eax
2805 ret
2806 SET_SIZE(lowbit)
2807
2808 #elif defined(__i386)
2809
2810 ENTRY(lowbit)
2811 bsfl 4(%esp), %eax
2812 jz 0f
2813 incl %eax
2814 ret
2815 0:
2816 xorl %eax, %eax
2817 ret
2818 SET_SIZE(lowbit)
2819
2820 #endif /* __i386 */
2821 #endif /* __lint */
2822
2823 #if defined(__lint)
2824
2825 /*ARGSUSED*/
2826 int
2827 highbit(ulong_t i)
2828 { return (0); }
2829
2830 /*ARGSUSED*/
2831 int
2832 highbit64(uint64_t i)
2833 { return (0); }
2834
2835 #else /* __lint */
2836
2837 #if defined(__amd64)
2838
2839 ENTRY(highbit)
2840 ALTENTRY(highbit64)
2841 movl $-1, %eax
2842 bsrq %rdi, %rdi
2843 cmovnz %edi, %eax
2844 incl %eax
2845 ret
2846 SET_SIZE(highbit64)
2847 SET_SIZE(highbit)
2848
2849 #elif defined(__i386)
2850
2851 ENTRY(highbit)
2852 bsrl 4(%esp), %eax
2853 jz 0f
2854 incl %eax
2855 ret
2856 0:
2857 xorl %eax, %eax
2858 ret
2859 SET_SIZE(highbit)
2860
2861 ENTRY(highbit64)
2862 bsrl 8(%esp), %eax
2863 jz highbit
2864 addl $33, %eax
2865 ret
2866 SET_SIZE(highbit64)
2867
2868 #endif /* __i386 */
2869 #endif /* __lint */
2870
2871 #if defined(__lint)
2872
2873 /*ARGSUSED*/
2874 uint64_t
2875 rdmsr(uint_t r)
2876 { return (0); }
2877
2878 /*ARGSUSED*/
2879 void
2880 wrmsr(uint_t r, const uint64_t val)
2881 {}
2882
2883 /*ARGSUSED*/
2884 uint64_t
2885 xrdmsr(uint_t r)
2886 { return (0); }
2887
2888 /*ARGSUSED*/
2889 void
2890 xwrmsr(uint_t r, const uint64_t val)
2891 {}
2892
2893 void
2894 invalidate_cache(void)
2895 {}
2896
2897 /*ARGSUSED*/
2898 uint64_t
2899 get_xcr(uint_t r)
2900 { return (0); }
2901
2902 /*ARGSUSED*/
2903 void
2904 set_xcr(uint_t r, const uint64_t val)
2905 {}
2906
2907 #else /* __lint */
2908
2909 #define XMSR_ACCESS_VAL $0x9c5a203a
2910
2911 #if defined(__amd64)
2912
2913 ENTRY(rdmsr)
2914 movl %edi, %ecx
2915 rdmsr
2916 shlq $32, %rdx
2917 orq %rdx, %rax
2918 ret
2919 SET_SIZE(rdmsr)
2920
2921 ENTRY(wrmsr)
2922 movq %rsi, %rdx
2923 shrq $32, %rdx
2924 movl %esi, %eax
2925 movl %edi, %ecx
2926 wrmsr
2927 ret
2928 SET_SIZE(wrmsr)
2929
2930 ENTRY(xrdmsr)
2931 pushq %rbp
2932 movq %rsp, %rbp
2954
2955 ENTRY(get_xcr)
2956 movl %edi, %ecx
2957 #xgetbv
2958 .byte 0x0f,0x01,0xd0
2959 shlq $32, %rdx
2960 orq %rdx, %rax
2961 ret
2962 SET_SIZE(get_xcr)
2963
2964 ENTRY(set_xcr)
2965 movq %rsi, %rdx
2966 shrq $32, %rdx
2967 movl %esi, %eax
2968 movl %edi, %ecx
2969 #xsetbv
2970 .byte 0x0f,0x01,0xd1
2971 ret
2972 SET_SIZE(set_xcr)
2973
2974 #elif defined(__i386)
2975
2976 ENTRY(rdmsr)
2977 movl 4(%esp), %ecx
2978 rdmsr
2979 ret
2980 SET_SIZE(rdmsr)
2981
2982 ENTRY(wrmsr)
2983 movl 4(%esp), %ecx
2984 movl 8(%esp), %eax
2985 movl 12(%esp), %edx
2986 wrmsr
2987 ret
2988 SET_SIZE(wrmsr)
2989
2990 ENTRY(xrdmsr)
2991 pushl %ebp
2992 movl %esp, %ebp
2993 movl 8(%esp), %ecx
2994 pushl %edi
2995 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2996 rdmsr
2997 popl %edi
2998 leave
2999 ret
3000 SET_SIZE(xrdmsr)
3001
3002 ENTRY(xwrmsr)
3003 pushl %ebp
3004 movl %esp, %ebp
3005 movl 8(%esp), %ecx
3006 movl 12(%esp), %eax
3007 movl 16(%esp), %edx
3008 pushl %edi
3009 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3010 wrmsr
3011 popl %edi
3012 leave
3013 ret
3014 SET_SIZE(xwrmsr)
3015
3016 ENTRY(get_xcr)
3017 movl 4(%esp), %ecx
3018 #xgetbv
3019 .byte 0x0f,0x01,0xd0
3020 ret
3021 SET_SIZE(get_xcr)
3022
3023 ENTRY(set_xcr)
3024 movl 4(%esp), %ecx
3025 movl 8(%esp), %eax
3026 movl 12(%esp), %edx
3027 #xsetbv
3028 .byte 0x0f,0x01,0xd1
3029 ret
3030 SET_SIZE(set_xcr)
3031
3032 #endif /* __i386 */
3033
3034 ENTRY(invalidate_cache)
3035 wbinvd
3036 ret
3037 SET_SIZE(invalidate_cache)
3038
3039 #endif /* __lint */
3040
3041 #if defined(__lint)
3042
3043 /*ARGSUSED*/
3044 void
3045 getcregs(struct cregs *crp)
3046 {}
3047
3048 #else /* __lint */
3049
3050 #if defined(__amd64)
3051
3052 ENTRY_NP(getcregs)
3053 #if defined(__xpv)
3054 /*
3055 * Only a few of the hardware control registers or descriptor tables
3056 * are directly accessible to us, so just zero the structure.
3057 *
3058 * XXPV Perhaps it would be helpful for the hypervisor to return
3059 * virtualized versions of these for post-mortem use.
3060 * (Need to reevaluate - perhaps it already does!)
3061 */
3062 pushq %rdi /* save *crp */
3063 movq $CREGSZ, %rsi
3064 call bzero
3065 popq %rdi
3066
3067 /*
3068 * Dump what limited information we can
3069 */
3070 movq %cr0, %rax
3071 movq %rax, CREG_CR0(%rdi) /* cr0 */
3094 movq %rax, CREG_TASKR(%rdi)
3095 str CREG_TASKR(%rdi) /* 2 bytes */
3096 movq %cr0, %rax
3097 movq %rax, CREG_CR0(%rdi) /* cr0 */
3098 movq %cr2, %rax
3099 movq %rax, CREG_CR2(%rdi) /* cr2 */
3100 movq %cr3, %rax
3101 movq %rax, CREG_CR3(%rdi) /* cr3 */
3102 movq %cr4, %rax
3103 movq %rax, CREG_CR4(%rdi) /* cr4 */
3104 movq %cr8, %rax
3105 movq %rax, CREG_CR8(%rdi) /* cr8 */
3106 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3107 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3108 #endif /* __xpv */
3109 ret
3110 SET_SIZE(getcregs)
3111
3112 #undef GETMSR
3113
3114 #elif defined(__i386)
3115
3116 ENTRY_NP(getcregs)
3117 #if defined(__xpv)
3118 /*
3119 * Only a few of the hardware control registers or descriptor tables
3120 * are directly accessible to us, so just zero the structure.
3121 *
3122 * XXPV Perhaps it would be helpful for the hypervisor to return
3123 * virtualized versions of these for post-mortem use.
3124 * (Need to reevaluate - perhaps it already does!)
3125 */
3126 movl 4(%esp), %edx
3127 pushl $CREGSZ
3128 pushl %edx
3129 call bzero
3130 addl $8, %esp
3131 movl 4(%esp), %edx
3132
3133 /*
3134 * Dump what limited information we can
3135 */
3136 movl %cr0, %eax
3137 movl %eax, CREG_CR0(%edx) /* cr0 */
3138 movl %cr2, %eax
3139 movl %eax, CREG_CR2(%edx) /* cr2 */
3140 movl %cr3, %eax
3141 movl %eax, CREG_CR3(%edx) /* cr3 */
3142 movl %cr4, %eax
3143 movl %eax, CREG_CR4(%edx) /* cr4 */
3144
3145 #else /* __xpv */
3146
3147 movl 4(%esp), %edx
3148 movw $0, CREG_GDT+6(%edx)
3149 movw $0, CREG_IDT+6(%edx)
3150 sgdt CREG_GDT(%edx) /* gdt */
3151 sidt CREG_IDT(%edx) /* idt */
3152 sldt CREG_LDT(%edx) /* ldt */
3153 str CREG_TASKR(%edx) /* task */
3154 movl %cr0, %eax
3155 movl %eax, CREG_CR0(%edx) /* cr0 */
3156 movl %cr2, %eax
3157 movl %eax, CREG_CR2(%edx) /* cr2 */
3158 movl %cr3, %eax
3159 movl %eax, CREG_CR3(%edx) /* cr3 */
3160 bt $X86FSET_LARGEPAGE, x86_featureset
3161 jnc .nocr4
3162 movl %cr4, %eax
3163 movl %eax, CREG_CR4(%edx) /* cr4 */
3164 jmp .skip
3165 .nocr4:
3166 movl $0, CREG_CR4(%edx)
3167 .skip:
3168 #endif
3169 ret
3170 SET_SIZE(getcregs)
3171
3172 #endif /* __i386 */
3173 #endif /* __lint */
3174
3175
3176 /*
3177 * A panic trigger is a word which is updated atomically and can only be set
3178 * once. We atomically store 0xDEFACEDD and load the old value. If the
3179 * previous value was 0, we succeed and return 1; otherwise return 0.
3180 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3181 * has its own version of this function to allow it to panic correctly from
3182 * probe context.
3183 */
3184 #if defined(__lint)
3185
3186 /*ARGSUSED*/
3187 int
3188 panic_trigger(int *tp)
3189 { return (0); }
3190
3191 /*ARGSUSED*/
3192 int
3193 dtrace_panic_trigger(int *tp)
3194 { return (0); }
3195
3196 #else /* __lint */
3197
3198 #if defined(__amd64)
3199
3200 ENTRY_NP(panic_trigger)
3201 xorl %eax, %eax
3202 movl $0xdefacedd, %edx
3203 lock
3204 xchgl %edx, (%rdi)
3205 cmpl $0, %edx
3206 je 0f
3207 movl $0, %eax
3208 ret
3209 0: movl $1, %eax
3210 ret
3211 SET_SIZE(panic_trigger)
3212
3213 ENTRY_NP(dtrace_panic_trigger)
3214 xorl %eax, %eax
3215 movl $0xdefacedd, %edx
3216 lock
3217 xchgl %edx, (%rdi)
3218 cmpl $0, %edx
3219 je 0f
3220 movl $0, %eax
3221 ret
3222 0: movl $1, %eax
3223 ret
3224 SET_SIZE(dtrace_panic_trigger)
3225
3226 #elif defined(__i386)
3227
3228 ENTRY_NP(panic_trigger)
3229 movl 4(%esp), %edx / %edx = address of trigger
3230 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3231 lock / assert lock
3232 xchgl %eax, (%edx) / exchange %eax and the trigger
3233 cmpl $0, %eax / if (%eax == 0x0)
3234 je 0f / return (1);
3235 movl $0, %eax / else
3236 ret / return (0);
3237 0: movl $1, %eax
3238 ret
3239 SET_SIZE(panic_trigger)
3240
3241 ENTRY_NP(dtrace_panic_trigger)
3242 movl 4(%esp), %edx / %edx = address of trigger
3243 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3244 lock / assert lock
3245 xchgl %eax, (%edx) / exchange %eax and the trigger
3246 cmpl $0, %eax / if (%eax == 0x0)
3247 je 0f / return (1);
3248 movl $0, %eax / else
3249 ret / return (0);
3250 0: movl $1, %eax
3251 ret
3252 SET_SIZE(dtrace_panic_trigger)
3253
3254 #endif /* __i386 */
3255 #endif /* __lint */
3256
3257 /*
3258 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3259 * into the panic code implemented in panicsys(). vpanic() is responsible
3260 * for passing through the format string and arguments, and constructing a
3261 * regs structure on the stack into which it saves the current register
3262 * values. If we are not dying due to a fatal trap, these registers will
3263 * then be preserved in panicbuf as the current processor state. Before
3264 * invoking panicsys(), vpanic() activates the first panic trigger (see
3265 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3266 * DTrace takes a slightly different panic path if it must panic from probe
3267 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3268 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3269 * branches back into vpanic().
3270 */
3271 #if defined(__lint)
3272
3273 /*ARGSUSED*/
3274 void
3275 vpanic(const char *format, va_list alist)
3276 {}
3277
3278 /*ARGSUSED*/
3279 void
3280 dtrace_vpanic(const char *format, va_list alist)
3281 {}
3282
3283 #else /* __lint */
3284
3285 #if defined(__amd64)
3286
3287 ENTRY_NP(vpanic) /* Initial stack layout: */
3288
3289 pushq %rbp /* | %rip | 0x60 */
3290 movq %rsp, %rbp /* | %rbp | 0x58 */
3291 pushfq /* | rfl | 0x50 */
3292 pushq %r11 /* | %r11 | 0x48 */
3293 pushq %r10 /* | %r10 | 0x40 */
3294 pushq %rbx /* | %rbx | 0x38 */
3295 pushq %rax /* | %rax | 0x30 */
3296 pushq %r9 /* | %r9 | 0x28 */
3297 pushq %r8 /* | %r8 | 0x20 */
3298 pushq %rcx /* | %rcx | 0x18 */
3299 pushq %rdx /* | %rdx | 0x10 */
3300 pushq %rsi /* | %rsi | 0x8 alist */
3301 pushq %rdi /* | %rdi | 0x0 format */
3302
3303 movq %rsp, %rbx /* %rbx = current %rsp */
3304
3305 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3306 call panic_trigger /* %eax = panic_trigger() */
3416 pushfq /* | rfl | 0x50 */
3417 pushq %r11 /* | %r11 | 0x48 */
3418 pushq %r10 /* | %r10 | 0x40 */
3419 pushq %rbx /* | %rbx | 0x38 */
3420 pushq %rax /* | %rax | 0x30 */
3421 pushq %r9 /* | %r9 | 0x28 */
3422 pushq %r8 /* | %r8 | 0x20 */
3423 pushq %rcx /* | %rcx | 0x18 */
3424 pushq %rdx /* | %rdx | 0x10 */
3425 pushq %rsi /* | %rsi | 0x8 alist */
3426 pushq %rdi /* | %rdi | 0x0 format */
3427
3428 movq %rsp, %rbx /* %rbx = current %rsp */
3429
3430 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3431 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3432 jmp vpanic_common
3433
3434 SET_SIZE(dtrace_vpanic)
3435
3436 #elif defined(__i386)
3437
3438 ENTRY_NP(vpanic) / Initial stack layout:
3439
3440 pushl %ebp / | %eip | 20
3441 movl %esp, %ebp / | %ebp | 16
3442 pushl %eax / | %eax | 12
3443 pushl %ebx / | %ebx | 8
3444 pushl %ecx / | %ecx | 4
3445 pushl %edx / | %edx | 0
3446
3447 movl %esp, %ebx / %ebx = current stack pointer
3448
3449 lea panic_quiesce, %eax / %eax = &panic_quiesce
3450 pushl %eax / push &panic_quiesce
3451 call panic_trigger / %eax = panic_trigger()
3452 addl $4, %esp / reset stack pointer
3453
3454 vpanic_common:
3455 cmpl $0, %eax / if (%eax == 0)
3456 je 0f / goto 0f;
3457
3458 /*
3459 * If panic_trigger() was successful, we are the first to initiate a
3460 * panic: we now switch to the reserved panic_stack before continuing.
3461 */
3462 lea panic_stack, %esp / %esp = panic_stack
3463 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3464
3465 0: subl $REGSIZE, %esp / allocate struct regs
3466
3467 /*
3468 * Now that we've got everything set up, store the register values as
3469 * they were when we entered vpanic() to the designated location in
3470 * the regs structure we allocated on the stack.
3471 */
3472 #if !defined(__GNUC_AS__)
3473 movw %gs, %edx
3474 movl %edx, REGOFF_GS(%esp)
3475 movw %fs, %edx
3476 movl %edx, REGOFF_FS(%esp)
3477 movw %es, %edx
3478 movl %edx, REGOFF_ES(%esp)
3479 movw %ds, %edx
3480 movl %edx, REGOFF_DS(%esp)
3481 #else /* __GNUC_AS__ */
3482 mov %gs, %edx
3483 mov %edx, REGOFF_GS(%esp)
3484 mov %fs, %edx
3485 mov %edx, REGOFF_FS(%esp)
3486 mov %es, %edx
3487 mov %edx, REGOFF_ES(%esp)
3488 mov %ds, %edx
3489 mov %edx, REGOFF_DS(%esp)
3490 #endif /* __GNUC_AS__ */
3491 movl %edi, REGOFF_EDI(%esp)
3492 movl %esi, REGOFF_ESI(%esp)
3493 movl 16(%ebx), %ecx
3494 movl %ecx, REGOFF_EBP(%esp)
3495 movl %ebx, %ecx
3496 addl $20, %ecx
3497 movl %ecx, REGOFF_ESP(%esp)
3498 movl 8(%ebx), %ecx
3499 movl %ecx, REGOFF_EBX(%esp)
3500 movl 0(%ebx), %ecx
3501 movl %ecx, REGOFF_EDX(%esp)
3502 movl 4(%ebx), %ecx
3503 movl %ecx, REGOFF_ECX(%esp)
3504 movl 12(%ebx), %ecx
3505 movl %ecx, REGOFF_EAX(%esp)
3506 movl $0, REGOFF_TRAPNO(%esp)
3507 movl $0, REGOFF_ERR(%esp)
3508 lea vpanic, %ecx
3509 movl %ecx, REGOFF_EIP(%esp)
3510 #if !defined(__GNUC_AS__)
3511 movw %cs, %edx
3512 #else /* __GNUC_AS__ */
3513 mov %cs, %edx
3514 #endif /* __GNUC_AS__ */
3515 movl %edx, REGOFF_CS(%esp)
3516 pushfl
3517 popl %ecx
3518 #if defined(__xpv)
3519 /*
3520 * Synthesize the PS_IE bit from the event mask bit
3521 */
3522 CURTHREAD(%edx)
3523 KPREEMPT_DISABLE(%edx)
3524 EVENT_MASK_TO_IE(%edx, %ecx)
3525 CURTHREAD(%edx)
3526 KPREEMPT_ENABLE_NOKP(%edx)
3527 #endif
3528 movl %ecx, REGOFF_EFL(%esp)
3529 movl $0, REGOFF_UESP(%esp)
3530 #if !defined(__GNUC_AS__)
3531 movw %ss, %edx
3532 #else /* __GNUC_AS__ */
3533 mov %ss, %edx
3534 #endif /* __GNUC_AS__ */
3535 movl %edx, REGOFF_SS(%esp)
3536
3537 movl %esp, %ecx / %ecx = ®s
3538 pushl %eax / push on_panic_stack
3539 pushl %ecx / push ®s
3540 movl 12(%ebp), %ecx / %ecx = alist
3541 pushl %ecx / push alist
3542 movl 8(%ebp), %ecx / %ecx = format
3543 pushl %ecx / push format
3544 call panicsys / panicsys();
3545 addl $16, %esp / pop arguments
3546
3547 addl $REGSIZE, %esp
3548 popl %edx
3549 popl %ecx
3550 popl %ebx
3551 popl %eax
3552 leave
3553 ret
3554 SET_SIZE(vpanic)
3555
3556 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3557
3558 pushl %ebp / | %eip | 20
3559 movl %esp, %ebp / | %ebp | 16
3560 pushl %eax / | %eax | 12
3561 pushl %ebx / | %ebx | 8
3562 pushl %ecx / | %ecx | 4
3563 pushl %edx / | %edx | 0
3564
3565 movl %esp, %ebx / %ebx = current stack pointer
3566
3567 lea panic_quiesce, %eax / %eax = &panic_quiesce
3568 pushl %eax / push &panic_quiesce
3569 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3570 addl $4, %esp / reset stack pointer
3571 jmp vpanic_common / jump back to common code
3572
3573 SET_SIZE(dtrace_vpanic)
3574
3575 #endif /* __i386 */
3576 #endif /* __lint */
3577
3578 #if defined(__lint)
3579
3580 void
3581 hres_tick(void)
3582 {}
3583
3584 int64_t timedelta;
3585 hrtime_t hrtime_base;
3586
3587 #else /* __lint */
3588
3589 DGDEF3(timedelta, 8, 8)
3590 .long 0, 0
3591
3592 /*
3593 * initialized to a non zero value to make pc_gethrtime()
3594 * work correctly even before clock is initialized
3595 */
3596 DGDEF3(hrtime_base, 8, 8)
3597 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3598
3599 DGDEF3(adj_shift, 4, 4)
3600 .long ADJ_SHIFT
3601
3602 #if defined(__amd64)
3603
3604 ENTRY_NP(hres_tick)
3605 pushq %rbp
3606 movq %rsp, %rbp
3607
3608 /*
3609 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3610 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3611 * At worst, performing this now instead of under CLOCK_LOCK may
3612 * introduce some jitter in pc_gethrestime().
3613 */
3614 movq gethrtimef(%rip), %rsi
3615 INDIRECT_CALL_REG(rsi)
3616 movq %rax, %r8
3617
3618 leaq hres_lock(%rip), %rax
3619 movb $-1, %dl
3620 .CL1:
3621 xchgb %dl, (%rax)
3622 testb %dl, %dl
3623 jz .CL3 /* got it */
3636 leaq hres_last_tick(%rip), %rax
3637 movq %r8, %r11
3638 subq (%rax), %r8
3639 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3640 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3641 /*
3642 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3643 */
3644 movq %r11, (%rax)
3645
3646 call __adj_hrestime
3647
3648 /*
3649 * release the hres_lock
3650 */
3651 incl hres_lock(%rip)
3652 leave
3653 ret
3654 SET_SIZE(hres_tick)
3655
3656 #elif defined(__i386)
3657
3658 ENTRY_NP(hres_tick)
3659 pushl %ebp
3660 movl %esp, %ebp
3661 pushl %esi
3662 pushl %ebx
3663
3664 /*
3665 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3666 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3667 * At worst, performing this now instead of under CLOCK_LOCK may
3668 * introduce some jitter in pc_gethrestime().
3669 */
3670 call *gethrtimef
3671 movl %eax, %ebx
3672 movl %edx, %esi
3673
3674 movl $hres_lock, %eax
3675 movl $-1, %edx
3676 .CL1:
3677 xchgb %dl, (%eax)
3678 testb %dl, %dl
3679 jz .CL3 / got it
3680 .CL2:
3681 cmpb $0, (%eax) / possible to get lock?
3682 pause
3683 jne .CL2
3684 jmp .CL1 / yes, try again
3685 .CL3:
3686 /*
3687 * compute the interval since last time hres_tick was called
3688 * and adjust hrtime_base and hrestime accordingly
3689 * hrtime_base is an 8 byte value (in nsec), hrestime is
3690 * timestruc_t (sec, nsec)
3691 */
3692
3693 lea hres_last_tick, %eax
3694
3695 movl %ebx, %edx
3696 movl %esi, %ecx
3697
3698 subl (%eax), %edx
3699 sbbl 4(%eax), %ecx
3700
3701 addl %edx, hrtime_base / add interval to hrtime_base
3702 adcl %ecx, hrtime_base+4
3703
3704 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3705
3706 /
3707 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3708 /
3709 movl %ebx, (%eax)
3710 movl %esi, 4(%eax)
3711
3712 / get hrestime at this moment. used as base for pc_gethrestime
3713 /
3714 / Apply adjustment, if any
3715 /
3716 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3717 / (max_hres_adj)
3718 /
3719 / void
3720 / adj_hrestime()
3721 / {
3722 / long long adj;
3723 /
3724 / if (hrestime_adj == 0)
3725 / adj = 0;
3726 / else if (hrestime_adj > 0) {
3727 / if (hrestime_adj < HRES_ADJ)
3728 / adj = hrestime_adj;
3729 / else
3730 / adj = HRES_ADJ;
3731 / }
3732 / else {
3733 / if (hrestime_adj < -(HRES_ADJ))
3734 / adj = -(HRES_ADJ);
3735 / else
3736 / adj = hrestime_adj;
3737 / }
3738 /
3739 / timedelta -= adj;
3740 / hrestime_adj = timedelta;
3741 / hrestime.tv_nsec += adj;
3742 /
3743 / while (hrestime.tv_nsec >= NANOSEC) {
3744 / one_sec++;
3745 / hrestime.tv_sec++;
3746 / hrestime.tv_nsec -= NANOSEC;
3747 / }
3748 / }
3749 __adj_hrestime:
3750 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3751 movl hrestime_adj+4, %edx
3752 andl %esi, %esi
3753 jne .CL4 / no
3754 andl %edx, %edx
3755 jne .CL4 / no
3756 subl %ecx, %ecx / yes, adj = 0;
3757 subl %edx, %edx
3758 jmp .CL5
3759 .CL4:
3760 subl %ecx, %ecx
3761 subl %eax, %eax
3762 subl %esi, %ecx
3763 sbbl %edx, %eax
3764 andl %eax, %eax / if (hrestime_adj > 0)
3765 jge .CL6
3766
3767 / In the following comments, HRES_ADJ is used, while in the code
3768 / max_hres_adj is used.
3769 /
3770 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3771 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3772 / on the logical equivalence of:
3773 /
3774 / !(hrestime_adj < HRES_ADJ)
3775 /
3776 / and the two step sequence:
3777 /
3778 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3779 /
3780 / which computes whether or not the least significant 32-bits
3781 / of hrestime_adj is greater than HRES_ADJ, followed by:
3782 /
3783 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3784 /
3785 / which generates a carry whenever step 1 is true or the most
3786 / significant long of the longlong hrestime_adj is non-zero.
3787
3788 movl max_hres_adj, %ecx / hrestime_adj is positive
3789 subl %esi, %ecx
3790 movl %edx, %eax
3791 adcl $-1, %eax
3792 jnc .CL7
3793 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3794 subl %edx, %edx
3795 jmp .CL5
3796
3797 / The following computation is similar to the one above.
3798 /
3799 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3800 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3801 / on the logical equivalence of:
3802 /
3803 / (hrestime_adj > -HRES_ADJ)
3804 /
3805 / and the two step sequence:
3806 /
3807 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3808 /
3809 / which means the least significant 32-bits of hrestime_adj is
3810 / greater than -HRES_ADJ, followed by:
3811 /
3812 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3813 /
3814 / which generates a carry only when step 1 is true and the most
3815 / significant long of the longlong hrestime_adj is -1.
3816
3817 .CL6: / hrestime_adj is negative
3818 movl %esi, %ecx
3819 addl max_hres_adj, %ecx
3820 movl %edx, %eax
3821 adcl $0, %eax
3822 jc .CL7
3823 xor %ecx, %ecx
3824 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3825 movl $-1, %edx
3826 jmp .CL5
3827 .CL7:
3828 movl %esi, %ecx / adj = hrestime_adj;
3829 .CL5:
3830 movl timedelta, %esi
3831 subl %ecx, %esi
3832 movl timedelta+4, %eax
3833 sbbl %edx, %eax
3834 movl %esi, timedelta
3835 movl %eax, timedelta+4 / timedelta -= adj;
3836 movl %esi, hrestime_adj
3837 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3838 addl hrestime+4, %ecx
3839
3840 movl %ecx, %eax / eax = tv_nsec
3841 1:
3842 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3843 jb .CL8 / no
3844 incl one_sec / yes, one_sec++;
3845 incl hrestime / hrestime.tv_sec++;
3846 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3847 jmp 1b / check for more seconds
3848
3849 .CL8:
3850 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3851 incl hres_lock / release the hres_lock
3852
3853 popl %ebx
3854 popl %esi
3855 leave
3856 ret
3857 SET_SIZE(hres_tick)
3858
3859 #endif /* __i386 */
3860 #endif /* __lint */
3861
3862 /*
3863 * void prefetch_smap_w(void *)
3864 *
3865 * Prefetch ahead within a linear list of smap structures.
3866 * Not implemented for ia32. Stub for compatibility.
3867 */
3868
3869 #if defined(__lint)
3870
3871 /*ARGSUSED*/
3872 void prefetch_smap_w(void *smp)
3873 {}
3874
3875 #else /* __lint */
3876
3877 ENTRY(prefetch_smap_w)
3878 rep; ret /* use 2 byte return instruction when branch target */
3879 /* AMD Software Optimization Guide - Section 6.2 */
3880 SET_SIZE(prefetch_smap_w)
3881
3882 #endif /* __lint */
3883
3884 /*
3885 * prefetch_page_r(page_t *)
3886 * issue prefetch instructions for a page_t
3887 */
3888 #if defined(__lint)
3889
3890 /*ARGSUSED*/
3891 void
3892 prefetch_page_r(void *pp)
3893 {}
3894
3895 #else /* __lint */
3896
3897 ENTRY(prefetch_page_r)
3898 rep; ret /* use 2 byte return instruction when branch target */
3899 /* AMD Software Optimization Guide - Section 6.2 */
3900 SET_SIZE(prefetch_page_r)
3901
3902 #endif /* __lint */
3903
3904 #if defined(__lint)
3905
3906 /*ARGSUSED*/
3907 int
3908 bcmp(const void *s1, const void *s2, size_t count)
3909 { return (0); }
3910
3911 #else /* __lint */
3912
3913 #if defined(__amd64)
3914
3915 ENTRY(bcmp)
3916 pushq %rbp
3917 movq %rsp, %rbp
3918 #ifdef DEBUG
3919 testq %rdx,%rdx
3920 je 1f
3921 movq postbootkernelbase(%rip), %r11
3922 cmpq %r11, %rdi
3923 jb 0f
3924 cmpq %r11, %rsi
3925 jnb 1f
3926 0: leaq .bcmp_panic_msg(%rip), %rdi
3927 xorl %eax, %eax
3928 call panic
3929 1:
3930 #endif /* DEBUG */
3931 call memcmp
3932 testl %eax, %eax
3933 setne %dl
3934 leave
3935 movzbl %dl, %eax
3936 ret
3937 SET_SIZE(bcmp)
3938
3939 #elif defined(__i386)
3940
3941 #define ARG_S1 8
3942 #define ARG_S2 12
3943 #define ARG_LENGTH 16
3944
3945 ENTRY(bcmp)
3946 pushl %ebp
3947 movl %esp, %ebp / create new stack frame
3948 #ifdef DEBUG
3949 cmpl $0, ARG_LENGTH(%ebp)
3950 je 1f
3951 movl postbootkernelbase, %eax
3952 cmpl %eax, ARG_S1(%ebp)
3953 jb 0f
3954 cmpl %eax, ARG_S2(%ebp)
3955 jnb 1f
3956 0: pushl $.bcmp_panic_msg
3957 call panic
3958 1:
3959 #endif /* DEBUG */
3960
3961 pushl %edi / save register variable
3962 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3963 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3964 cmpl %eax, %ecx / if the same string
3965 je .equal / goto .equal
3966 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3967 cmpl $4, %edi / if %edi < 4
3968 jb .byte_check / goto .byte_check
3969 .align 4
3970 .word_loop:
3971 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3972 leal -4(%edi), %edi / %edi -= 4
3973 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3974 jne .word_not_equal / if not equal, goto .word_not_equal
3975 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3976 leal 4(%eax), %eax / %eax += 4 (next word)
3977 cmpl $4, %edi / if %edi >= 4
3978 jae .word_loop / goto .word_loop
3979 .byte_check:
3980 cmpl $0, %edi / if %edi == 0
3981 je .equal / goto .equal
3982 jmp .byte_loop / goto .byte_loop (checks in bytes)
3983 .word_not_equal:
3984 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3985 .align 4
3986 .byte_loop:
3987 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3988 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3989 jne .not_equal / if not equal, goto .not_equal
3990 incl %ecx / %ecx++ (next byte)
3991 incl %eax / %eax++ (next byte)
3992 decl %edi / %edi--
3993 jnz .byte_loop / if not zero, goto .byte_loop
3994 .equal:
3995 xorl %eax, %eax / %eax = 0
3996 popl %edi / restore register variable
3997 leave / restore old stack frame
3998 ret / return (NULL)
3999 .align 4
4000 .not_equal:
4001 movl $1, %eax / return 1
4002 popl %edi / restore register variable
4003 leave / restore old stack frame
4004 ret / return (NULL)
4005 SET_SIZE(bcmp)
4006
4007 #endif /* __i386 */
4008
4009 #ifdef DEBUG
4010 .text
4011 .bcmp_panic_msg:
4012 .string "bcmp: arguments below kernelbase"
4013 #endif /* DEBUG */
4014
4015 #endif /* __lint */
4016
4017 #if defined(__lint)
4018
4019 uint_t
4020 bsrw_insn(uint16_t mask)
4021 {
4022 uint_t index = sizeof (mask) * NBBY - 1;
4023
4024 while ((mask & (1 << index)) == 0)
4025 index--;
4026 return (index);
4027 }
4028
4029 #else /* __lint */
4030
4031 #if defined(__amd64)
4032
4033 ENTRY_NP(bsrw_insn)
4034 xorl %eax, %eax
4035 bsrw %di, %ax
4036 ret
4037 SET_SIZE(bsrw_insn)
4038
4039 #elif defined(__i386)
4040
4041 ENTRY_NP(bsrw_insn)
4042 movw 4(%esp), %cx
4043 xorl %eax, %eax
4044 bsrw %cx, %ax
4045 ret
4046 SET_SIZE(bsrw_insn)
4047
4048 #endif /* __i386 */
4049 #endif /* __lint */
4050
4051 #if defined(__lint)
4052
4053 uint_t
4054 atomic_btr32(uint32_t *pending, uint_t pil)
4055 {
4056 return (*pending &= ~(1 << pil));
4057 }
4058
4059 #else /* __lint */
4060
4061 #if defined(__i386)
4062
4063 ENTRY_NP(atomic_btr32)
4064 movl 4(%esp), %ecx
4065 movl 8(%esp), %edx
4066 xorl %eax, %eax
4067 lock
4068 btrl %edx, (%ecx)
4069 setc %al
4070 ret
4071 SET_SIZE(atomic_btr32)
4072
4073 #endif /* __i386 */
4074 #endif /* __lint */
4075
4076 #if defined(__lint)
4077
4078 /*ARGSUSED*/
4079 void
4080 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4081 uint_t arg2)
4082 {}
4083
4084 #else /* __lint */
4085
4086 #if defined(__amd64)
4087
4088 ENTRY_NP(switch_sp_and_call)
4089 pushq %rbp
4090 movq %rsp, %rbp /* set up stack frame */
4091 movq %rdi, %rsp /* switch stack pointer */
4092 movq %rdx, %rdi /* pass func arg 1 */
4093 movq %rsi, %r11 /* save function to call */
4094 movq %rcx, %rsi /* pass func arg 2 */
4095 INDIRECT_CALL_REG(r11) /* call function */
4096 leave /* restore stack */
4097 ret
4098 SET_SIZE(switch_sp_and_call)
4099
4100 #elif defined(__i386)
4101
4102 ENTRY_NP(switch_sp_and_call)
4103 pushl %ebp
4104 mov %esp, %ebp /* set up stack frame */
4105 movl 8(%ebp), %esp /* switch stack pointer */
4106 pushl 20(%ebp) /* push func arg 2 */
4107 pushl 16(%ebp) /* push func arg 1 */
4108 call *12(%ebp) /* call function */
4109 addl $8, %esp /* pop arguments */
4110 leave /* restore stack */
4111 ret
4112 SET_SIZE(switch_sp_and_call)
4113
4114 #endif /* __i386 */
4115 #endif /* __lint */
4116
4117 #if defined(__lint)
4118
4119 void
4120 kmdb_enter(void)
4121 {}
4122
4123 #else /* __lint */
4124
4125 #if defined(__amd64)
4126
4127 ENTRY_NP(kmdb_enter)
4128 pushq %rbp
4129 movq %rsp, %rbp
4130
4131 /*
4132 * Save flags, do a 'cli' then return the saved flags
4133 */
4134 call intr_clear
4135
4136 int $T_DBGENTR
4137
4138 /*
4139 * Restore the saved flags
4140 */
4141 movq %rax, %rdi
4142 call intr_restore
4143
4144 leave
4145 ret
4146 SET_SIZE(kmdb_enter)
4147
4148 #elif defined(__i386)
4149
4150 ENTRY_NP(kmdb_enter)
4151 pushl %ebp
4152 movl %esp, %ebp
4153
4154 /*
4155 * Save flags, do a 'cli' then return the saved flags
4156 */
4157 call intr_clear
4158
4159 int $T_DBGENTR
4160
4161 /*
4162 * Restore the saved flags
4163 */
4164 pushl %eax
4165 call intr_restore
4166 addl $4, %esp
4167
4168 leave
4169 ret
4170 SET_SIZE(kmdb_enter)
4171
4172 #endif /* __i386 */
4173 #endif /* __lint */
4174
4175 #if defined(__lint)
4176
4177 void
4178 return_instr(void)
4179 {}
4180
4181 #else /* __lint */
4182
4183 ENTRY_NP(return_instr)
4184 rep; ret /* use 2 byte instruction when branch target */
4185 /* AMD Software Optimization Guide - Section 6.2 */
4186 SET_SIZE(return_instr)
4187
4188 #endif /* __lint */
4189
4190 #if defined(__lint)
4191
4192 ulong_t
4193 getflags(void)
4194 {
4195 return (0);
4196 }
4197
4198 #else /* __lint */
4199
4200 #if defined(__amd64)
4201
4202 ENTRY(getflags)
4203 pushfq
4204 popq %rax
4205 #if defined(__xpv)
4206 CURTHREAD(%rdi)
4207 KPREEMPT_DISABLE(%rdi)
4208 /*
4209 * Synthesize the PS_IE bit from the event mask bit
4210 */
4211 CURVCPU(%r11)
4212 andq $_BITNOT(PS_IE), %rax
4213 XEN_TEST_UPCALL_MASK(%r11)
4214 jnz 1f
4215 orq $PS_IE, %rax
4216 1:
4217 KPREEMPT_ENABLE_NOKP(%rdi)
4218 #endif
4219 ret
4220 SET_SIZE(getflags)
4221
4222 #elif defined(__i386)
4223
4224 ENTRY(getflags)
4225 pushfl
4226 popl %eax
4227 #if defined(__xpv)
4228 CURTHREAD(%ecx)
4229 KPREEMPT_DISABLE(%ecx)
4230 /*
4231 * Synthesize the PS_IE bit from the event mask bit
4232 */
4233 CURVCPU(%edx)
4234 andl $_BITNOT(PS_IE), %eax
4235 XEN_TEST_UPCALL_MASK(%edx)
4236 jnz 1f
4237 orl $PS_IE, %eax
4238 1:
4239 KPREEMPT_ENABLE_NOKP(%ecx)
4240 #endif
4241 ret
4242 SET_SIZE(getflags)
4243
4244 #endif /* __i386 */
4245
4246 #endif /* __lint */
4247
4248 #if defined(__lint)
4249
4250 ftrace_icookie_t
4251 ftrace_interrupt_disable(void)
4252 { return (0); }
4253
4254 #else /* __lint */
4255
4256 #if defined(__amd64)
4257
4258 ENTRY(ftrace_interrupt_disable)
4259 pushfq
4260 popq %rax
4261 CLI(%rdx)
4262 ret
4263 SET_SIZE(ftrace_interrupt_disable)
4264
4265 #elif defined(__i386)
4266
4267 ENTRY(ftrace_interrupt_disable)
4268 pushfl
4269 popl %eax
4270 CLI(%edx)
4271 ret
4272 SET_SIZE(ftrace_interrupt_disable)
4273
4274 #endif /* __i386 */
4275 #endif /* __lint */
4276
4277 #if defined(__lint)
4278
4279 /*ARGSUSED*/
4280 void
4281 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4282 {}
4283
4284 #else /* __lint */
4285
4286 #if defined(__amd64)
4287
4288 ENTRY(ftrace_interrupt_enable)
4289 pushq %rdi
4290 popfq
4291 ret
4292 SET_SIZE(ftrace_interrupt_enable)
4293
4294 #elif defined(__i386)
4295
4296 ENTRY(ftrace_interrupt_enable)
4297 movl 4(%esp), %eax
4298 pushl %eax
4299 popfl
4300 ret
4301 SET_SIZE(ftrace_interrupt_enable)
4302
4303 #endif /* __i386 */
4304 #endif /* __lint */
4305
4306 #if defined (__lint)
4307
4308 /*ARGSUSED*/
4309 void
4310 clflush_insn(caddr_t addr)
4311 {}
4312
4313 #else /* __lint */
4314
4315 #if defined (__amd64)
4316 ENTRY(clflush_insn)
4317 clflush (%rdi)
4318 ret
4319 SET_SIZE(clflush_insn)
4320 #elif defined (__i386)
4321 ENTRY(clflush_insn)
4322 movl 4(%esp), %eax
4323 clflush (%eax)
4324 ret
4325 SET_SIZE(clflush_insn)
4326
4327 #endif /* __i386 */
4328 #endif /* __lint */
4329
4330 #if defined (__lint)
4331 /*ARGSUSED*/
4332 void
4333 mfence_insn(void)
4334 {}
4335
4336 #else /* __lint */
4337
4338 #if defined (__amd64)
4339 ENTRY(mfence_insn)
4340 mfence
4341 ret
4342 SET_SIZE(mfence_insn)
4343 #elif defined (__i386)
4344 ENTRY(mfence_insn)
4345 mfence
4346 ret
4347 SET_SIZE(mfence_insn)
4348
4349 #endif /* __i386 */
4350 #endif /* __lint */
4351
4352 /*
4353 * VMware implements an I/O port that programs can query to detect if software
4354 * is running in a VMware hypervisor. This hypervisor port behaves differently
4355 * depending on magic values in certain registers and modifies some registers
4356 * as a side effect.
4357 *
4358 * References: http://kb.vmware.com/kb/1009458
4359 */
4360
4361 #if defined(__lint)
4362
4363 /* ARGSUSED */
4364 void
4365 vmware_port(int cmd, uint32_t *regs) { return; }
4366
4367 #else
4368
4369 #if defined(__amd64)
4370
4371 ENTRY(vmware_port)
4372 pushq %rbx
4373 movl $VMWARE_HVMAGIC, %eax
4374 movl $0xffffffff, %ebx
4375 movl %edi, %ecx
4376 movl $VMWARE_HVPORT, %edx
4377 inl (%dx)
4378 movl %eax, (%rsi)
4379 movl %ebx, 4(%rsi)
4380 movl %ecx, 8(%rsi)
4381 movl %edx, 12(%rsi)
4382 popq %rbx
4383 ret
4384 SET_SIZE(vmware_port)
4385
4386 #elif defined(__i386)
4387
4388 ENTRY(vmware_port)
4389 pushl %ebx
4390 pushl %esi
4391 movl $VMWARE_HVMAGIC, %eax
4392 movl $0xffffffff, %ebx
4393 movl 12(%esp), %ecx
4394 movl $VMWARE_HVPORT, %edx
4395 inl (%dx)
4396 movl 16(%esp), %esi
4397 movl %eax, (%esi)
4398 movl %ebx, 4(%esi)
4399 movl %ecx, 8(%esi)
4400 movl %edx, 12(%esi)
4401 popl %esi
4402 popl %ebx
4403 ret
4404 SET_SIZE(vmware_port)
4405
4406 #endif /* __i386 */
4407 #endif /* __lint */
|
40 /*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
57 #include <sys/psw.h>
58 #include <sys/x86_archext.h>
59
60 #include "assym.h"
61 #include <sys/dditypes.h>
62
63 /*
64 * on_fault()
65 *
66 * Catch lofault faults. Like setjmp except it returns one
67 * if code following causes uncorrectable fault. Turned off
68 * by calling no_fault(). Note that while under on_fault(),
69 * SMAP is disabled. For more information see
70 * uts/intel/ia32/ml/copy.s.
71 */
72
73 ENTRY(on_fault)
74 movq %gs:CPU_THREAD, %rsi
75 leaq catch_fault(%rip), %rdx
76 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
77 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
78 call smap_disable /* allow user accesses */
79 jmp setjmp /* let setjmp do the rest */
80
81 catch_fault:
82 movq %gs:CPU_THREAD, %rsi
83 movq T_ONFAULT(%rsi), %rdi /* address of save area */
84 xorl %eax, %eax
85 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
86 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
87 call smap_enable /* disallow user accesses */
88 jmp longjmp /* let longjmp do the rest */
89 SET_SIZE(on_fault)
90
91 ENTRY(no_fault)
92 movq %gs:CPU_THREAD, %rsi
93 xorl %eax, %eax
94 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
95 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
96 call smap_enable /* disallow user accesses */
97 ret
98 SET_SIZE(no_fault)
99
100 /*
101 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
102 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
103 */
104
105 ENTRY(on_trap_trampoline)
106 movq %gs:CPU_THREAD, %rsi
107 movq T_ONTRAP(%rsi), %rdi
108 addq $OT_JMPBUF, %rdi
109 jmp longjmp
110 SET_SIZE(on_trap_trampoline)
111
112 /*
113 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
114 * more information about the on_trap() mechanism. If the on_trap_data is the
115 * same as the topmost stack element, we just modify that element.
116 */
117
118 ENTRY(on_trap)
119 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
120 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
121 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
122 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
123 xorl %ecx, %ecx
124 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
125 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
126 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
127 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
128 cmpq %rdi, %rcx /* if (otp == %rcx) */
129 je 0f /* don't modify t_ontrap */
130
131 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
132 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
133
134 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
135 jmp setjmp
136 SET_SIZE(on_trap)
137
138 /*
139 * Setjmp and longjmp implement non-local gotos using state vectors
140 * type label_t.
141 */
142
143 #if LABEL_PC != 0
144 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
145 #endif /* LABEL_PC != 0 */
146
147 ENTRY(setjmp)
148 movq %rsp, LABEL_SP(%rdi)
149 movq %rbp, LABEL_RBP(%rdi)
150 movq %rbx, LABEL_RBX(%rdi)
151 movq %r12, LABEL_R12(%rdi)
152 movq %r13, LABEL_R13(%rdi)
153 movq %r14, LABEL_R14(%rdi)
154 movq %r15, LABEL_R15(%rdi)
155 movq (%rsp), %rdx /* return address */
156 movq %rdx, (%rdi) /* LABEL_PC is 0 */
157 xorl %eax, %eax /* return 0 */
158 ret
159 SET_SIZE(setjmp)
160
161 ENTRY(longjmp)
162 movq LABEL_SP(%rdi), %rsp
163 movq LABEL_RBP(%rdi), %rbp
164 movq LABEL_RBX(%rdi), %rbx
165 movq LABEL_R12(%rdi), %r12
166 movq LABEL_R13(%rdi), %r13
167 movq LABEL_R14(%rdi), %r14
168 movq LABEL_R15(%rdi), %r15
169 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
170 movq %rdx, (%rsp)
171 xorl %eax, %eax
172 incl %eax /* return 1 */
173 ret
174 SET_SIZE(longjmp)
175
176 /*
177 * if a() calls b() calls caller(),
178 * caller() returns return address in a().
179 * (Note: We assume a() and b() are C routines which do the normal entry/exit
180 * sequence.)
181 */
182
183 ENTRY(caller)
184 movq 8(%rbp), %rax /* b()'s return pc, in a() */
185 ret
186 SET_SIZE(caller)
187
188 /*
189 * if a() calls callee(), callee() returns the
190 * return address in a();
191 */
192
193 ENTRY(callee)
194 movq (%rsp), %rax /* callee()'s return pc, in a() */
195 ret
196 SET_SIZE(callee)
197
198 /*
199 * return the current frame pointer
200 */
201
202 ENTRY(getfp)
203 movq %rbp, %rax
204 ret
205 SET_SIZE(getfp)
206
207 /*
208 * Invalidate a single page table entry in the TLB
209 */
210
211 ENTRY(mmu_invlpg)
212 invlpg (%rdi)
213 ret
214 SET_SIZE(mmu_invlpg)
215
216
217 /*
218 * Get/Set the value of various control registers
219 */
220
221 ENTRY(getcr0)
222 movq %cr0, %rax
223 ret
224 SET_SIZE(getcr0)
225
226 ENTRY(setcr0)
227 movq %rdi, %cr0
228 ret
229 SET_SIZE(setcr0)
230
231 ENTRY(getcr2)
232 #if defined(__xpv)
233 movq %gs:CPU_VCPU_INFO, %rax
234 movq VCPU_INFO_ARCH_CR2(%rax), %rax
235 #else
236 movq %cr2, %rax
237 #endif
238 ret
239 SET_SIZE(getcr2)
240
261 ENTRY(getcr4)
262 movq %cr4, %rax
263 ret
264 SET_SIZE(getcr4)
265
266 ENTRY(setcr4)
267 movq %rdi, %cr4
268 ret
269 SET_SIZE(setcr4)
270
271 ENTRY(getcr8)
272 movq %cr8, %rax
273 ret
274 SET_SIZE(getcr8)
275
276 ENTRY(setcr8)
277 movq %rdi, %cr8
278 ret
279 SET_SIZE(setcr8)
280
281 ENTRY(__cpuid_insn)
282 movq %rbx, %r8
283 movq %rcx, %r9
284 movq %rdx, %r11
285 movl (%rdi), %eax /* %eax = regs->cp_eax */
286 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
287 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
288 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
289 cpuid
290 movl %eax, (%rdi) /* regs->cp_eax = %eax */
291 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
292 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
293 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
294 movq %r8, %rbx
295 movq %r9, %rcx
296 movq %r11, %rdx
297 ret
298 SET_SIZE(__cpuid_insn)
299
300 ENTRY_NP(i86_monitor)
301 pushq %rbp
302 movq %rsp, %rbp
303 movq %rdi, %rax /* addr */
304 movq %rsi, %rcx /* extensions */
305 /* rdx contains input arg3: hints */
306 clflush (%rax)
307 .byte 0x0f, 0x01, 0xc8 /* monitor */
308 leave
309 ret
310 SET_SIZE(i86_monitor)
311
312 ENTRY_NP(i86_mwait)
313 pushq %rbp
314 call x86_md_clear
315 movq %rsp, %rbp
316 movq %rdi, %rax /* data */
317 movq %rsi, %rcx /* extensions */
318 .byte 0x0f, 0x01, 0xc9 /* mwait */
319 leave
320 ret
321 SET_SIZE(i86_mwait)
322
323 #if defined(__xpv)
324 /*
325 * Defined in C
326 */
327 #else
328
329 ENTRY_NP(tsc_read)
330 movq %rbx, %r11
331 movl $0, %eax
332 cpuid
333 rdtsc
334 movq %r11, %rbx
335 shlq $32, %rdx
336 orq %rdx, %rax
337 ret
338 .globl _tsc_mfence_start
339 _tsc_mfence_start:
340 mfence
341 rdtsc
342 shlq $32, %rdx
343 orq %rdx, %rax
344 ret
345 .globl _tsc_mfence_end
346 _tsc_mfence_end:
347 .globl _tscp_start
348 _tscp_start:
353 .globl _tscp_end
354 _tscp_end:
355 .globl _no_rdtsc_start
356 _no_rdtsc_start:
357 xorl %edx, %edx
358 xorl %eax, %eax
359 ret
360 .globl _no_rdtsc_end
361 _no_rdtsc_end:
362 .globl _tsc_lfence_start
363 _tsc_lfence_start:
364 lfence
365 rdtsc
366 shlq $32, %rdx
367 orq %rdx, %rax
368 ret
369 .globl _tsc_lfence_end
370 _tsc_lfence_end:
371 SET_SIZE(tsc_read)
372
373
374 #endif /* __xpv */
375
376 ENTRY_NP(randtick)
377 rdtsc
378 shlq $32, %rdx
379 orq %rdx, %rax
380 ret
381 SET_SIZE(randtick)
382 /*
383 * Insert entryp after predp in a doubly linked list.
384 */
385
386 ENTRY(_insque)
387 movq (%rsi), %rax /* predp->forw */
388 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
389 movq %rax, (%rdi) /* entryp->forw = predp->forw */
390 movq %rdi, (%rsi) /* predp->forw = entryp */
391 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
392 ret
393 SET_SIZE(_insque)
394
395 /*
396 * Remove entryp from a doubly linked list
397 */
398
399 ENTRY(_remque)
400 movq (%rdi), %rax /* entry->forw */
401 movq CPTRSIZE(%rdi), %rdx /* entry->back */
402 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
403 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
404 ret
405 SET_SIZE(_remque)
406
407 /*
408 * Returns the number of
409 * non-NULL bytes in string argument.
410 */
411
412 /*
413 * This is close to a simple transliteration of a C version of this
414 * routine. We should either just -make- this be a C version, or
415 * justify having it in assembler by making it significantly faster.
416 *
417 * size_t
418 * strlen(const char *s)
419 * {
420 * const char *s0;
421 * #if defined(DEBUG)
422 * if ((uintptr_t)s < KERNELBASE)
423 * panic(.str_panic_msg);
424 * #endif
425 * for (s0 = s; *s; s++)
426 * ;
427 * return (s - s0);
428 * }
429 */
430
431 ENTRY(strlen)
437 movq %rsp, %rbp
438 leaq .str_panic_msg(%rip), %rdi
439 xorl %eax, %eax
440 call panic
441 #endif /* DEBUG */
442 str_valid:
443 cmpb $0, (%rdi)
444 movq %rdi, %rax
445 je .null_found
446 .align 4
447 .strlen_loop:
448 incq %rdi
449 cmpb $0, (%rdi)
450 jne .strlen_loop
451 .null_found:
452 subq %rax, %rdi
453 movq %rdi, %rax
454 ret
455 SET_SIZE(strlen)
456
457 #ifdef DEBUG
458 .text
459 .str_panic_msg:
460 .string "strlen: argument below kernelbase"
461 #endif /* DEBUG */
462
463 /*
464 * Berkeley 4.3 introduced symbolically named interrupt levels
465 * as a way deal with priority in a machine independent fashion.
466 * Numbered priorities are machine specific, and should be
467 * discouraged where possible.
468 *
469 * Note, for the machine specific priorities there are
470 * examples listed for devices that use a particular priority.
471 * It should not be construed that all devices of that
472 * type should be at that priority. It is currently were
473 * the current devices fit into the priority scheme based
474 * upon time criticalness.
475 *
476 * The underlying assumption of these assignments is that
477 * IPL 10 is the highest level from which a device
478 * routine can call wakeup. Devices that interrupt from higher
479 * levels are restricted in what they can do. If they need
480 * kernels services they should schedule a routine at a lower
481 * level (via software interrupt) to do the required
482 * processing.
483 *
484 * Examples of this higher usage:
485 * Level Usage
486 * 14 Profiling clock (and PROM uart polling clock)
487 * 12 Serial ports
488 *
489 * The serial ports request lower level processing on level 6.
490 *
491 * Also, almost all splN routines (where N is a number or a
492 * mnemonic) will do a RAISE(), on the assumption that they are
493 * never used to lower our priority.
494 * The exceptions are:
495 * spl8() Because you can't be above 15 to begin with!
496 * splzs() Because this is used at boot time to lower our
497 * priority, to allow the PROM to poll the uart.
498 * spl0() Used to lower priority to 0.
499 */
500
501 #define SETPRI(level) \
502 movl $/**/level, %edi; /* new priority */ \
503 jmp do_splx /* redirect to do_splx */
504
505 #define RAISE(level) \
506 movl $/**/level, %edi; /* new priority */ \
507 jmp splr /* redirect to splr */
508
509 /* locks out all interrupts, including memory errors */
510 ENTRY(spl8)
511 SETPRI(15)
512 SET_SIZE(spl8)
513
514 /* just below the level that profiling runs */
515 ENTRY(spl7)
516 RAISE(13)
517 SET_SIZE(spl7)
518
519 /* sun specific - highest priority onboard serial i/o asy ports */
520 ENTRY(splzs)
521 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
522 SET_SIZE(splzs)
523
524 ENTRY(splhi)
525 ALTENTRY(splhigh)
526 ALTENTRY(spl6)
527 ALTENTRY(i_ddi_splhigh)
528
529 RAISE(DISP_LEVEL)
530
531 SET_SIZE(i_ddi_splhigh)
532 SET_SIZE(spl6)
533 SET_SIZE(splhigh)
534 SET_SIZE(splhi)
535
536 /* allow all interrupts */
537 ENTRY(spl0)
538 SETPRI(0)
539 SET_SIZE(spl0)
540
541
542 /* splx implementation */
543 ENTRY(splx)
544 jmp do_splx /* redirect to common splx code */
545 SET_SIZE(splx)
546
547 ENTRY(wait_500ms)
548 pushq %rbx
549 movl $50000, %ebx
550 1:
551 call tenmicrosec
552 decl %ebx
553 jnz 1b
554 popq %rbx
555 ret
556 SET_SIZE(wait_500ms)
557
558 #define RESET_METHOD_KBC 1
559 #define RESET_METHOD_PORT92 2
560 #define RESET_METHOD_PCI 4
561
562 DGDEF3(pc_reset_methods, 4, 8)
563 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
564
565 ENTRY(pc_reset)
566
567 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
568 jz 1f
569
570 /
571 / Try the classic keyboard controller-triggered reset.
572 /
573 movw $0x64, %dx
574 movb $0xfe, %al
575 outb (%dx)
576
577 / Wait up to 500 milliseconds here for the keyboard controller
578 / to pull the reset line. On some systems where the keyboard
579 / controller is slow to pull the reset line, the next reset method
580 / may be executed (which may be bad if those systems hang when the
581 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
582 / and Ferrari 4000 (doesn't like the cf9 reset method))
583
584 call wait_500ms
585
586 1:
587 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
588 jz 3f
589
590 /
591 / Try port 0x92 fast reset
592 /
593 movw $0x92, %dx
594 inb (%dx)
595 cmpb $0xff, %al / If port's not there, we should get back 0xFF
596 je 1f
597 testb $1, %al / If bit 0
598 jz 2f / is clear, jump to perform the reset
599 andb $0xfe, %al / otherwise,
600 outb (%dx) / clear bit 0 first, then
601 2:
602 orb $1, %al / Set bit 0
603 outb (%dx) / and reset the system
604 1:
605
606 call wait_500ms
607
608 3:
609 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
610 jz 4f
611
612 / Try the PCI (soft) reset vector (should work on all modern systems,
613 / but has been shown to cause problems on 450NX systems, and some newer
614 / systems (e.g. ATI IXP400-equipped systems))
615 / When resetting via this method, 2 writes are required. The first
616 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
617 / power cycle).
618 / The reset occurs on the second write, during bit 2 's transition from
619 / 0->1.
620 movw $0xcf9, %dx
621 movb $0x2, %al / Reset mode = hard, no power cycle
622 outb (%dx)
623 movb $0x6, %al
624 outb (%dx)
625
626 call wait_500ms
627
628 4:
629 /
630 / port 0xcf9 failed also. Last-ditch effort is to
631 / triple-fault the CPU.
632 / Also, use triple fault for EFI firmware
633 /
634 ENTRY(efi_reset)
635 pushq $0x0
636 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
637 lidt (%rsp)
638 int $0x0 / Trigger interrupt, generate triple-fault
639
640 cli
641 hlt / Wait forever
642 /*NOTREACHED*/
643 SET_SIZE(efi_reset)
644 SET_SIZE(pc_reset)
645
646 /*
647 * C callable in and out routines
648 */
649
650 ENTRY(outl)
651 movw %di, %dx
652 movl %esi, %eax
653 outl (%dx)
654 ret
655 SET_SIZE(outl)
656
657 ENTRY(outw)
658 movw %di, %dx
659 movw %si, %ax
660 D16 outl (%dx) /* XX64 why not outw? */
661 ret
662 SET_SIZE(outw)
663
664 ENTRY(outb)
665 movw %di, %dx
666 movb %sil, %al
667 outb (%dx)
668 ret
669 SET_SIZE(outb)
670
671 ENTRY(inl)
672 xorl %eax, %eax
673 movw %di, %dx
674 inl (%dx)
675 ret
676 SET_SIZE(inl)
677
678 ENTRY(inw)
679 xorl %eax, %eax
680 movw %di, %dx
681 D16 inl (%dx)
682 ret
683 SET_SIZE(inw)
684
685
686 ENTRY(inb)
687 xorl %eax, %eax
688 movw %di, %dx
689 inb (%dx)
690 ret
691 SET_SIZE(inb)
692
693
694 ENTRY(repoutsw)
695 movl %edx, %ecx
696 movw %di, %dx
697 rep
698 D16 outsl
699 ret
700 SET_SIZE(repoutsw)
701
702
703 ENTRY(repinsw)
704 movl %edx, %ecx
705 movw %di, %dx
706 rep
707 D16 insl
708 ret
709 SET_SIZE(repinsw)
710
711
712 ENTRY(repinsb)
713 movl %edx, %ecx
714 movw %di, %dx
715 movq %rsi, %rdi
716 rep
717 insb
718 ret
719 SET_SIZE(repinsb)
720
721
722 /*
723 * Input a stream of 32-bit words.
724 * NOTE: count is a DWORD count.
725 */
726
727 ENTRY(repinsd)
728 movl %edx, %ecx
729 movw %di, %dx
730 movq %rsi, %rdi
731 rep
732 insl
733 ret
734 SET_SIZE(repinsd)
735
736 /*
737 * Output a stream of bytes
738 * NOTE: count is a byte count
739 */
740
741 ENTRY(repoutsb)
742 movl %edx, %ecx
743 movw %di, %dx
744 rep
745 outsb
746 ret
747 SET_SIZE(repoutsb)
748
749 /*
750 * Output a stream of 32-bit words
751 * NOTE: count is a DWORD count
752 */
753
754 ENTRY(repoutsd)
755 movl %edx, %ecx
756 movw %di, %dx
757 rep
758 outsl
759 ret
760 SET_SIZE(repoutsd)
761
762 /*
763 * void int3(void)
764 * void int18(void)
765 * void int20(void)
766 * void int_cmci(void)
767 */
768
769 ENTRY(int3)
770 int $T_BPTFLT
771 ret
772 SET_SIZE(int3)
773
774 ENTRY(int18)
775 int $T_MCE
776 ret
777 SET_SIZE(int18)
778
779 ENTRY(int20)
780 movl boothowto, %eax
781 andl $RB_DEBUG, %eax
782 jz 1f
783
784 int $T_DBGENTR
785 1:
786 rep; ret /* use 2 byte return instruction when branch target */
787 /* AMD Software Optimization Guide - Section 6.2 */
788 SET_SIZE(int20)
789
790 ENTRY(int_cmci)
791 int $T_ENOEXTFLT
792 ret
793 SET_SIZE(int_cmci)
794
795 ENTRY(scanc)
796 /* rdi == size */
797 /* rsi == cp */
798 /* rdx == table */
799 /* rcx == mask */
800 addq %rsi, %rdi /* end = &cp[size] */
801 .scanloop:
802 cmpq %rdi, %rsi /* while (cp < end */
803 jnb .scandone
804 movzbq (%rsi), %r8 /* %r8 = *cp */
805 incq %rsi /* cp++ */
806 testb %cl, (%r8, %rdx)
807 jz .scanloop /* && (table[*cp] & mask) == 0) */
808 decq %rsi /* (fix post-increment) */
809 .scandone:
810 movl %edi, %eax
811 subl %esi, %eax /* return (end - cp) */
812 ret
813 SET_SIZE(scanc)
814
815 /*
816 * Replacement functions for ones that are normally inlined.
817 * In addition to the copy in i86.il, they are defined here just in case.
818 */
819
820 ENTRY(intr_clear)
821 ENTRY(clear_int_flag)
822 pushfq
823 popq %rax
824 #if defined(__xpv)
825 leaq xpv_panicking, %rdi
826 movl (%rdi), %edi
827 cmpl $0, %edi
828 jne 2f
829 CLIRET(%rdi, %dl) /* returns event mask in %dl */
830 /*
831 * Synthesize the PS_IE bit from the event mask bit
832 */
833 andq $_BITNOT(PS_IE), %rax
834 testb $1, %dl
835 jnz 1f
836 orq $PS_IE, %rax
837 1:
838 ret
839 2:
840 #endif
841 CLI(%rdi)
842 ret
843 SET_SIZE(clear_int_flag)
844 SET_SIZE(intr_clear)
845
846 ENTRY(curcpup)
847 movq %gs:CPU_SELF, %rax
848 ret
849 SET_SIZE(curcpup)
850
851 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
852 * These functions reverse the byte order of the input parameter and returns
853 * the result. This is to convert the byte order from host byte order
854 * (little endian) to network byte order (big endian), or vice versa.
855 */
856
857 ENTRY(htonll)
858 ALTENTRY(ntohll)
859 movq %rdi, %rax
860 bswapq %rax
861 ret
862 SET_SIZE(ntohll)
863 SET_SIZE(htonll)
864
865 /* XX64 there must be shorter sequences for this */
866 ENTRY(htonl)
867 ALTENTRY(ntohl)
868 movl %edi, %eax
869 bswap %eax
870 ret
871 SET_SIZE(ntohl)
872 SET_SIZE(htonl)
873
874 /* XX64 there must be better sequences for this */
875 ENTRY(htons)
876 ALTENTRY(ntohs)
877 movl %edi, %eax
878 bswap %eax
879 shrl $16, %eax
880 ret
881 SET_SIZE(ntohs)
882 SET_SIZE(htons)
883
884
885 ENTRY(intr_restore)
886 ENTRY(restore_int_flag)
887 testq $PS_IE, %rdi
888 jz 1f
889 #if defined(__xpv)
890 leaq xpv_panicking, %rsi
891 movl (%rsi), %esi
892 cmpl $0, %esi
893 jne 1f
894 /*
895 * Since we're -really- running unprivileged, our attempt
896 * to change the state of the IF bit will be ignored.
897 * The virtual IF bit is tweaked by CLI and STI.
898 */
899 IE_TO_EVENT_MASK(%rsi, %rdi)
900 #else
901 sti
902 #endif
903 1:
904 ret
905 SET_SIZE(restore_int_flag)
906 SET_SIZE(intr_restore)
907
908 ENTRY(sti)
909 STI
910 ret
911 SET_SIZE(sti)
912
913 ENTRY(cli)
914 CLI(%rax)
915 ret
916 SET_SIZE(cli)
917
918 ENTRY(dtrace_interrupt_disable)
919 pushfq
920 popq %rax
921 #if defined(__xpv)
922 leaq xpv_panicking, %rdi
923 movl (%rdi), %edi
924 cmpl $0, %edi
925 jne .dtrace_interrupt_disable_done
926 CLIRET(%rdi, %dl) /* returns event mask in %dl */
927 /*
928 * Synthesize the PS_IE bit from the event mask bit
929 */
930 andq $_BITNOT(PS_IE), %rax
931 testb $1, %dl
932 jnz .dtrace_interrupt_disable_done
933 orq $PS_IE, %rax
934 #else
935 CLI(%rdx)
936 #endif
937 .dtrace_interrupt_disable_done:
938 ret
939 SET_SIZE(dtrace_interrupt_disable)
940
941 ENTRY(dtrace_interrupt_enable)
942 pushq %rdi
943 popfq
944 #if defined(__xpv)
945 leaq xpv_panicking, %rdx
946 movl (%rdx), %edx
947 cmpl $0, %edx
948 jne .dtrace_interrupt_enable_done
949 /*
950 * Since we're -really- running unprivileged, our attempt
951 * to change the state of the IF bit will be ignored. The
952 * virtual IF bit is tweaked by CLI and STI.
953 */
954 IE_TO_EVENT_MASK(%rdx, %rdi)
955 #endif
956 .dtrace_interrupt_enable_done:
957 ret
958 SET_SIZE(dtrace_interrupt_enable)
959
960
961 ENTRY(dtrace_membar_producer)
962 rep; ret /* use 2 byte return instruction when branch target */
963 /* AMD Software Optimization Guide - Section 6.2 */
964 SET_SIZE(dtrace_membar_producer)
965
966 ENTRY(dtrace_membar_consumer)
967 rep; ret /* use 2 byte return instruction when branch target */
968 /* AMD Software Optimization Guide - Section 6.2 */
969 SET_SIZE(dtrace_membar_consumer)
970
971 ENTRY(threadp)
972 movq %gs:CPU_THREAD, %rax
973 ret
974 SET_SIZE(threadp)
975
976 /*
977 * Checksum routine for Internet Protocol Headers
978 */
979
980 ENTRY(ip_ocsum)
981 pushq %rbp
982 movq %rsp, %rbp
983 #ifdef DEBUG
984 movq postbootkernelbase(%rip), %rax
985 cmpq %rax, %rdi
986 jnb 1f
987 xorl %eax, %eax
988 movq %rdi, %rsi
989 leaq .ip_ocsum_panic_msg(%rip), %rdi
990 call panic
991 /*NOTREACHED*/
992 .ip_ocsum_panic_msg:
993 .string "ip_ocsum: address 0x%p below kernelbase\n"
994 1:
995 #endif
996 movl %esi, %ecx /* halfword_count */
997 movq %rdi, %rsi /* address */
998 /* partial sum in %edx */
999 xorl %eax, %eax
1078 .size_aligned:
1079 movl %ecx, %edi
1080 shrl $1, %ecx
1081 shl $1, %edi
1082 subq $64, %rdi
1083 addq %rdi, %rsi
1084 leaq .ip_ocsum_jmptbl(%rip), %rdi
1085 leaq (%rdi, %rcx, 8), %rdi
1086 xorl %ecx, %ecx
1087 clc
1088 movq (%rdi), %rdi
1089 INDIRECT_JMP_REG(rdi)
1090
1091 .align 8
1092 .ip_ocsum_jmptbl:
1093 .quad .only0, .only4, .only8, .only12, .only16, .only20
1094 .quad .only24, .only28, .only32, .only36, .only40, .only44
1095 .quad .only48, .only52, .only56, .only60
1096 SET_SIZE(ip_ocsum)
1097
1098 /*
1099 * multiply two long numbers and yield a u_longlong_t result, callable from C.
1100 * Provided to manipulate hrtime_t values.
1101 */
1102
1103 ENTRY(mul32)
1104 xorl %edx, %edx /* XX64 joe, paranoia? */
1105 movl %edi, %eax
1106 mull %esi
1107 shlq $32, %rdx
1108 orq %rdx, %rax
1109 ret
1110 SET_SIZE(mul32)
1111
1112 ENTRY(scan_memory)
1113 shrq $3, %rsi /* convert %rsi from byte to quadword count */
1114 jz .scanm_done
1115 movq %rsi, %rcx /* move count into rep control register */
1116 movq %rdi, %rsi /* move addr into lodsq control reg. */
1117 rep lodsq /* scan the memory range */
1118 .scanm_done:
1119 rep; ret /* use 2 byte return instruction when branch target */
1120 /* AMD Software Optimization Guide - Section 6.2 */
1121 SET_SIZE(scan_memory)
1122
1123
1124 ENTRY(lowbit)
1125 movl $-1, %eax
1126 bsfq %rdi, %rdi
1127 cmovnz %edi, %eax
1128 incl %eax
1129 ret
1130 SET_SIZE(lowbit)
1131
1132 ENTRY(highbit)
1133 ALTENTRY(highbit64)
1134 movl $-1, %eax
1135 bsrq %rdi, %rdi
1136 cmovnz %edi, %eax
1137 incl %eax
1138 ret
1139 SET_SIZE(highbit64)
1140 SET_SIZE(highbit)
1141
1142 #define XMSR_ACCESS_VAL $0x9c5a203a
1143
1144 ENTRY(rdmsr)
1145 movl %edi, %ecx
1146 rdmsr
1147 shlq $32, %rdx
1148 orq %rdx, %rax
1149 ret
1150 SET_SIZE(rdmsr)
1151
1152 ENTRY(wrmsr)
1153 movq %rsi, %rdx
1154 shrq $32, %rdx
1155 movl %esi, %eax
1156 movl %edi, %ecx
1157 wrmsr
1158 ret
1159 SET_SIZE(wrmsr)
1160
1161 ENTRY(xrdmsr)
1162 pushq %rbp
1163 movq %rsp, %rbp
1185
1186 ENTRY(get_xcr)
1187 movl %edi, %ecx
1188 #xgetbv
1189 .byte 0x0f,0x01,0xd0
1190 shlq $32, %rdx
1191 orq %rdx, %rax
1192 ret
1193 SET_SIZE(get_xcr)
1194
1195 ENTRY(set_xcr)
1196 movq %rsi, %rdx
1197 shrq $32, %rdx
1198 movl %esi, %eax
1199 movl %edi, %ecx
1200 #xsetbv
1201 .byte 0x0f,0x01,0xd1
1202 ret
1203 SET_SIZE(set_xcr)
1204
1205 ENTRY(invalidate_cache)
1206 wbinvd
1207 ret
1208 SET_SIZE(invalidate_cache)
1209
1210 ENTRY_NP(getcregs)
1211 #if defined(__xpv)
1212 /*
1213 * Only a few of the hardware control registers or descriptor tables
1214 * are directly accessible to us, so just zero the structure.
1215 *
1216 * XXPV Perhaps it would be helpful for the hypervisor to return
1217 * virtualized versions of these for post-mortem use.
1218 * (Need to reevaluate - perhaps it already does!)
1219 */
1220 pushq %rdi /* save *crp */
1221 movq $CREGSZ, %rsi
1222 call bzero
1223 popq %rdi
1224
1225 /*
1226 * Dump what limited information we can
1227 */
1228 movq %cr0, %rax
1229 movq %rax, CREG_CR0(%rdi) /* cr0 */
1252 movq %rax, CREG_TASKR(%rdi)
1253 str CREG_TASKR(%rdi) /* 2 bytes */
1254 movq %cr0, %rax
1255 movq %rax, CREG_CR0(%rdi) /* cr0 */
1256 movq %cr2, %rax
1257 movq %rax, CREG_CR2(%rdi) /* cr2 */
1258 movq %cr3, %rax
1259 movq %rax, CREG_CR3(%rdi) /* cr3 */
1260 movq %cr4, %rax
1261 movq %rax, CREG_CR4(%rdi) /* cr4 */
1262 movq %cr8, %rax
1263 movq %rax, CREG_CR8(%rdi) /* cr8 */
1264 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
1265 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
1266 #endif /* __xpv */
1267 ret
1268 SET_SIZE(getcregs)
1269
1270 #undef GETMSR
1271
1272
1273 /*
1274 * A panic trigger is a word which is updated atomically and can only be set
1275 * once. We atomically store 0xDEFACEDD and load the old value. If the
1276 * previous value was 0, we succeed and return 1; otherwise return 0.
1277 * This allows a partially corrupt trigger to still trigger correctly. DTrace
1278 * has its own version of this function to allow it to panic correctly from
1279 * probe context.
1280 */
1281
1282 ENTRY_NP(panic_trigger)
1283 xorl %eax, %eax
1284 movl $0xdefacedd, %edx
1285 lock
1286 xchgl %edx, (%rdi)
1287 cmpl $0, %edx
1288 je 0f
1289 movl $0, %eax
1290 ret
1291 0: movl $1, %eax
1292 ret
1293 SET_SIZE(panic_trigger)
1294
1295 ENTRY_NP(dtrace_panic_trigger)
1296 xorl %eax, %eax
1297 movl $0xdefacedd, %edx
1298 lock
1299 xchgl %edx, (%rdi)
1300 cmpl $0, %edx
1301 je 0f
1302 movl $0, %eax
1303 ret
1304 0: movl $1, %eax
1305 ret
1306 SET_SIZE(dtrace_panic_trigger)
1307
1308 /*
1309 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
1310 * into the panic code implemented in panicsys(). vpanic() is responsible
1311 * for passing through the format string and arguments, and constructing a
1312 * regs structure on the stack into which it saves the current register
1313 * values. If we are not dying due to a fatal trap, these registers will
1314 * then be preserved in panicbuf as the current processor state. Before
1315 * invoking panicsys(), vpanic() activates the first panic trigger (see
1316 * common/os/panic.c) and switches to the panic_stack if successful. Note that
1317 * DTrace takes a slightly different panic path if it must panic from probe
1318 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
1319 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
1320 * branches back into vpanic().
1321 */
1322
1323 ENTRY_NP(vpanic) /* Initial stack layout: */
1324
1325 pushq %rbp /* | %rip | 0x60 */
1326 movq %rsp, %rbp /* | %rbp | 0x58 */
1327 pushfq /* | rfl | 0x50 */
1328 pushq %r11 /* | %r11 | 0x48 */
1329 pushq %r10 /* | %r10 | 0x40 */
1330 pushq %rbx /* | %rbx | 0x38 */
1331 pushq %rax /* | %rax | 0x30 */
1332 pushq %r9 /* | %r9 | 0x28 */
1333 pushq %r8 /* | %r8 | 0x20 */
1334 pushq %rcx /* | %rcx | 0x18 */
1335 pushq %rdx /* | %rdx | 0x10 */
1336 pushq %rsi /* | %rsi | 0x8 alist */
1337 pushq %rdi /* | %rdi | 0x0 format */
1338
1339 movq %rsp, %rbx /* %rbx = current %rsp */
1340
1341 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
1342 call panic_trigger /* %eax = panic_trigger() */
1452 pushfq /* | rfl | 0x50 */
1453 pushq %r11 /* | %r11 | 0x48 */
1454 pushq %r10 /* | %r10 | 0x40 */
1455 pushq %rbx /* | %rbx | 0x38 */
1456 pushq %rax /* | %rax | 0x30 */
1457 pushq %r9 /* | %r9 | 0x28 */
1458 pushq %r8 /* | %r8 | 0x20 */
1459 pushq %rcx /* | %rcx | 0x18 */
1460 pushq %rdx /* | %rdx | 0x10 */
1461 pushq %rsi /* | %rsi | 0x8 alist */
1462 pushq %rdi /* | %rdi | 0x0 format */
1463
1464 movq %rsp, %rbx /* %rbx = current %rsp */
1465
1466 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
1467 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
1468 jmp vpanic_common
1469
1470 SET_SIZE(dtrace_vpanic)
1471
1472 DGDEF3(timedelta, 8, 8)
1473 .long 0, 0
1474
1475 /*
1476 * initialized to a non zero value to make pc_gethrtime()
1477 * work correctly even before clock is initialized
1478 */
1479 DGDEF3(hrtime_base, 8, 8)
1480 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
1481
1482 DGDEF3(adj_shift, 4, 4)
1483 .long ADJ_SHIFT
1484
1485 ENTRY_NP(hres_tick)
1486 pushq %rbp
1487 movq %rsp, %rbp
1488
1489 /*
1490 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
1491 * hres_last_tick can only be modified while holding CLOCK_LOCK).
1492 * At worst, performing this now instead of under CLOCK_LOCK may
1493 * introduce some jitter in pc_gethrestime().
1494 */
1495 movq gethrtimef(%rip), %rsi
1496 INDIRECT_CALL_REG(rsi)
1497 movq %rax, %r8
1498
1499 leaq hres_lock(%rip), %rax
1500 movb $-1, %dl
1501 .CL1:
1502 xchgb %dl, (%rax)
1503 testb %dl, %dl
1504 jz .CL3 /* got it */
1517 leaq hres_last_tick(%rip), %rax
1518 movq %r8, %r11
1519 subq (%rax), %r8
1520 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
1521 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
1522 /*
1523 * Now that we have CLOCK_LOCK, we can update hres_last_tick
1524 */
1525 movq %r11, (%rax)
1526
1527 call __adj_hrestime
1528
1529 /*
1530 * release the hres_lock
1531 */
1532 incl hres_lock(%rip)
1533 leave
1534 ret
1535 SET_SIZE(hres_tick)
1536
1537 /*
1538 * void prefetch_smap_w(void *)
1539 *
1540 * Prefetch ahead within a linear list of smap structures.
1541 * Not implemented for ia32. Stub for compatibility.
1542 */
1543
1544 ENTRY(prefetch_smap_w)
1545 rep; ret /* use 2 byte return instruction when branch target */
1546 /* AMD Software Optimization Guide - Section 6.2 */
1547 SET_SIZE(prefetch_smap_w)
1548
1549 /*
1550 * prefetch_page_r(page_t *)
1551 * issue prefetch instructions for a page_t
1552 */
1553
1554 ENTRY(prefetch_page_r)
1555 rep; ret /* use 2 byte return instruction when branch target */
1556 /* AMD Software Optimization Guide - Section 6.2 */
1557 SET_SIZE(prefetch_page_r)
1558
1559 ENTRY(bcmp)
1560 pushq %rbp
1561 movq %rsp, %rbp
1562 #ifdef DEBUG
1563 testq %rdx,%rdx
1564 je 1f
1565 movq postbootkernelbase(%rip), %r11
1566 cmpq %r11, %rdi
1567 jb 0f
1568 cmpq %r11, %rsi
1569 jnb 1f
1570 0: leaq .bcmp_panic_msg(%rip), %rdi
1571 xorl %eax, %eax
1572 call panic
1573 1:
1574 #endif /* DEBUG */
1575 call memcmp
1576 testl %eax, %eax
1577 setne %dl
1578 leave
1579 movzbl %dl, %eax
1580 ret
1581 SET_SIZE(bcmp)
1582
1583 #ifdef DEBUG
1584 .text
1585 .bcmp_panic_msg:
1586 .string "bcmp: arguments below kernelbase"
1587 #endif /* DEBUG */
1588
1589 ENTRY_NP(bsrw_insn)
1590 xorl %eax, %eax
1591 bsrw %di, %ax
1592 ret
1593 SET_SIZE(bsrw_insn)
1594
1595 ENTRY_NP(switch_sp_and_call)
1596 pushq %rbp
1597 movq %rsp, %rbp /* set up stack frame */
1598 movq %rdi, %rsp /* switch stack pointer */
1599 movq %rdx, %rdi /* pass func arg 1 */
1600 movq %rsi, %r11 /* save function to call */
1601 movq %rcx, %rsi /* pass func arg 2 */
1602 INDIRECT_CALL_REG(r11) /* call function */
1603 leave /* restore stack */
1604 ret
1605 SET_SIZE(switch_sp_and_call)
1606
1607 ENTRY_NP(kmdb_enter)
1608 pushq %rbp
1609 movq %rsp, %rbp
1610
1611 /*
1612 * Save flags, do a 'cli' then return the saved flags
1613 */
1614 call intr_clear
1615
1616 int $T_DBGENTR
1617
1618 /*
1619 * Restore the saved flags
1620 */
1621 movq %rax, %rdi
1622 call intr_restore
1623
1624 leave
1625 ret
1626 SET_SIZE(kmdb_enter)
1627
1628 ENTRY_NP(return_instr)
1629 rep; ret /* use 2 byte instruction when branch target */
1630 /* AMD Software Optimization Guide - Section 6.2 */
1631 SET_SIZE(return_instr)
1632
1633 ENTRY(getflags)
1634 pushfq
1635 popq %rax
1636 #if defined(__xpv)
1637 CURTHREAD(%rdi)
1638 KPREEMPT_DISABLE(%rdi)
1639 /*
1640 * Synthesize the PS_IE bit from the event mask bit
1641 */
1642 CURVCPU(%r11)
1643 andq $_BITNOT(PS_IE), %rax
1644 XEN_TEST_UPCALL_MASK(%r11)
1645 jnz 1f
1646 orq $PS_IE, %rax
1647 1:
1648 KPREEMPT_ENABLE_NOKP(%rdi)
1649 #endif
1650 ret
1651 SET_SIZE(getflags)
1652
1653 ENTRY(ftrace_interrupt_disable)
1654 pushfq
1655 popq %rax
1656 CLI(%rdx)
1657 ret
1658 SET_SIZE(ftrace_interrupt_disable)
1659
1660 ENTRY(ftrace_interrupt_enable)
1661 pushq %rdi
1662 popfq
1663 ret
1664 SET_SIZE(ftrace_interrupt_enable)
1665
1666 ENTRY(clflush_insn)
1667 clflush (%rdi)
1668 ret
1669 SET_SIZE(clflush_insn)
1670
1671 ENTRY(mfence_insn)
1672 mfence
1673 ret
1674 SET_SIZE(mfence_insn)
1675
1676 /*
1677 * VMware implements an I/O port that programs can query to detect if software
1678 * is running in a VMware hypervisor. This hypervisor port behaves differently
1679 * depending on magic values in certain registers and modifies some registers
1680 * as a side effect.
1681 *
1682 * References: http://kb.vmware.com/kb/1009458
1683 */
1684
1685 ENTRY(vmware_port)
1686 pushq %rbx
1687 movl $VMWARE_HVMAGIC, %eax
1688 movl $0xffffffff, %ebx
1689 movl %edi, %ecx
1690 movl $VMWARE_HVPORT, %edx
1691 inl (%dx)
1692 movl %eax, (%rsi)
1693 movl %ebx, 4(%rsi)
1694 movl %ecx, 8(%rsi)
1695 movl %edx, 12(%rsi)
1696 popq %rbx
1697 ret
1698 SET_SIZE(vmware_port)
1699
|