1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 * Copyright 2019 Joyent, Inc.
27 */
28
29 /*
30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 * All Rights Reserved
33 */
34
35 /*
36 * Copyright (c) 2009, Intel Corporation.
37 * All rights reserved.
38 */
39
40 /*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
57 #include <sys/psw.h>
58 #include <sys/x86_archext.h>
59
60 #if defined(__lint)
61 #include <sys/types.h>
62 #include <sys/systm.h>
63 #include <sys/thread.h>
64 #include <sys/archsystm.h>
65 #include <sys/byteorder.h>
66 #include <sys/dtrace.h>
67 #include <sys/ftrace.h>
68 #else /* __lint */
69 #include "assym.h"
70 #endif /* __lint */
71 #include <sys/dditypes.h>
72
73 /*
74 * on_fault()
75 *
76 * Catch lofault faults. Like setjmp except it returns one
77 * if code following causes uncorrectable fault. Turned off
78 * by calling no_fault(). Note that while under on_fault(),
79 * SMAP is disabled. For more information see
80 * uts/intel/ia32/ml/copy.s.
81 */
82
83 #if defined(__lint)
84
85 /* ARGSUSED */
86 int
87 on_fault(label_t *ljb)
88 { return (0); }
89
90 void
91 no_fault(void)
92 {}
93
94 #else /* __lint */
95
96 #if defined(__amd64)
97
98 ENTRY(on_fault)
99 movq %gs:CPU_THREAD, %rsi
100 leaq catch_fault(%rip), %rdx
101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
103 call smap_disable /* allow user accesses */
104 jmp setjmp /* let setjmp do the rest */
105
106 catch_fault:
107 movq %gs:CPU_THREAD, %rsi
108 movq T_ONFAULT(%rsi), %rdi /* address of save area */
109 xorl %eax, %eax
110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
112 call smap_enable /* disallow user accesses */
113 jmp longjmp /* let longjmp do the rest */
114 SET_SIZE(on_fault)
115
116 ENTRY(no_fault)
117 movq %gs:CPU_THREAD, %rsi
118 xorl %eax, %eax
119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
121 call smap_enable /* disallow user accesses */
122 ret
123 SET_SIZE(no_fault)
124
125 #elif defined(__i386)
126
127 ENTRY(on_fault)
128 movl %gs:CPU_THREAD, %edx
129 movl 4(%esp), %eax /* jumpbuf address */
130 leal catch_fault, %ecx
131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
133 jmp setjmp /* let setjmp do the rest */
134
135 catch_fault:
136 movl %gs:CPU_THREAD, %edx
137 xorl %eax, %eax
138 movl T_ONFAULT(%edx), %ecx /* address of save area */
139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
141 pushl %ecx
142 call longjmp /* let longjmp do the rest */
143 SET_SIZE(on_fault)
144
145 ENTRY(no_fault)
146 movl %gs:CPU_THREAD, %edx
147 xorl %eax, %eax
148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
150 ret
151 SET_SIZE(no_fault)
152
153 #endif /* __i386 */
154 #endif /* __lint */
155
156 /*
157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 */
160
161 #if defined(lint)
162
163 void
164 on_trap_trampoline(void)
165 {}
166
167 #else /* __lint */
168
169 #if defined(__amd64)
170
171 ENTRY(on_trap_trampoline)
172 movq %gs:CPU_THREAD, %rsi
173 movq T_ONTRAP(%rsi), %rdi
174 addq $OT_JMPBUF, %rdi
175 jmp longjmp
176 SET_SIZE(on_trap_trampoline)
177
178 #elif defined(__i386)
179
180 ENTRY(on_trap_trampoline)
181 movl %gs:CPU_THREAD, %eax
182 movl T_ONTRAP(%eax), %eax
183 addl $OT_JMPBUF, %eax
184 pushl %eax
185 call longjmp
186 SET_SIZE(on_trap_trampoline)
187
188 #endif /* __i386 */
189 #endif /* __lint */
190
191 /*
192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
193 * more information about the on_trap() mechanism. If the on_trap_data is the
194 * same as the topmost stack element, we just modify that element.
195 */
196 #if defined(lint)
197
198 /*ARGSUSED*/
199 int
200 on_trap(on_trap_data_t *otp, uint_t prot)
201 { return (0); }
202
203 #else /* __lint */
204
205 #if defined(__amd64)
206
207 ENTRY(on_trap)
208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
212 xorl %ecx, %ecx
213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
217 cmpq %rdi, %rcx /* if (otp == %rcx) */
218 je 0f /* don't modify t_ontrap */
219
220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
222
223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
224 jmp setjmp
225 SET_SIZE(on_trap)
226
227 #elif defined(__i386)
228
229 ENTRY(on_trap)
230 movl 4(%esp), %eax /* %eax = otp */
231 movl 8(%esp), %edx /* %edx = prot */
232
233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
241 cmpl %eax, %ecx /* if (otp == %ecx) */
242 je 0f /* don't modify t_ontrap */
243
244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
246
247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
248 movl %eax, 4(%esp) /* put %eax back on the stack */
249 jmp setjmp /* let setjmp do the rest */
250 SET_SIZE(on_trap)
251
252 #endif /* __i386 */
253 #endif /* __lint */
254
255 /*
256 * Setjmp and longjmp implement non-local gotos using state vectors
257 * type label_t.
258 */
259
260 #if defined(__lint)
261
262 /* ARGSUSED */
263 int
264 setjmp(label_t *lp)
265 { return (0); }
266
267 /* ARGSUSED */
268 void
269 longjmp(label_t *lp)
270 {}
271
272 #else /* __lint */
273
274 #if LABEL_PC != 0
275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276 #endif /* LABEL_PC != 0 */
277
278 #if defined(__amd64)
279
280 ENTRY(setjmp)
281 movq %rsp, LABEL_SP(%rdi)
282 movq %rbp, LABEL_RBP(%rdi)
283 movq %rbx, LABEL_RBX(%rdi)
284 movq %r12, LABEL_R12(%rdi)
285 movq %r13, LABEL_R13(%rdi)
286 movq %r14, LABEL_R14(%rdi)
287 movq %r15, LABEL_R15(%rdi)
288 movq (%rsp), %rdx /* return address */
289 movq %rdx, (%rdi) /* LABEL_PC is 0 */
290 xorl %eax, %eax /* return 0 */
291 ret
292 SET_SIZE(setjmp)
293
294 ENTRY(longjmp)
295 movq LABEL_SP(%rdi), %rsp
296 movq LABEL_RBP(%rdi), %rbp
297 movq LABEL_RBX(%rdi), %rbx
298 movq LABEL_R12(%rdi), %r12
299 movq LABEL_R13(%rdi), %r13
300 movq LABEL_R14(%rdi), %r14
301 movq LABEL_R15(%rdi), %r15
302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
303 movq %rdx, (%rsp)
304 xorl %eax, %eax
305 incl %eax /* return 1 */
306 ret
307 SET_SIZE(longjmp)
308
309 #elif defined(__i386)
310
311 ENTRY(setjmp)
312 movl 4(%esp), %edx /* address of save area */
313 movl %ebp, LABEL_EBP(%edx)
314 movl %ebx, LABEL_EBX(%edx)
315 movl %esi, LABEL_ESI(%edx)
316 movl %edi, LABEL_EDI(%edx)
317 movl %esp, 4(%edx)
318 movl (%esp), %ecx /* %eip (return address) */
319 movl %ecx, (%edx) /* LABEL_PC is 0 */
320 subl %eax, %eax /* return 0 */
321 ret
322 SET_SIZE(setjmp)
323
324 ENTRY(longjmp)
325 movl 4(%esp), %edx /* address of save area */
326 movl LABEL_EBP(%edx), %ebp
327 movl LABEL_EBX(%edx), %ebx
328 movl LABEL_ESI(%edx), %esi
329 movl LABEL_EDI(%edx), %edi
330 movl 4(%edx), %esp
331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
332 movl $1, %eax
333 addl $4, %esp /* pop ret adr */
334 jmp *%ecx /* indirect */
335 SET_SIZE(longjmp)
336
337 #endif /* __i386 */
338 #endif /* __lint */
339
340 /*
341 * if a() calls b() calls caller(),
342 * caller() returns return address in a().
343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 * sequence.)
345 */
346
347 #if defined(__lint)
348
349 caddr_t
350 caller(void)
351 { return (0); }
352
353 #else /* __lint */
354
355 #if defined(__amd64)
356
357 ENTRY(caller)
358 movq 8(%rbp), %rax /* b()'s return pc, in a() */
359 ret
360 SET_SIZE(caller)
361
362 #elif defined(__i386)
363
364 ENTRY(caller)
365 movl 4(%ebp), %eax /* b()'s return pc, in a() */
366 ret
367 SET_SIZE(caller)
368
369 #endif /* __i386 */
370 #endif /* __lint */
371
372 /*
373 * if a() calls callee(), callee() returns the
374 * return address in a();
375 */
376
377 #if defined(__lint)
378
379 caddr_t
380 callee(void)
381 { return (0); }
382
383 #else /* __lint */
384
385 #if defined(__amd64)
386
387 ENTRY(callee)
388 movq (%rsp), %rax /* callee()'s return pc, in a() */
389 ret
390 SET_SIZE(callee)
391
392 #elif defined(__i386)
393
394 ENTRY(callee)
395 movl (%esp), %eax /* callee()'s return pc, in a() */
396 ret
397 SET_SIZE(callee)
398
399 #endif /* __i386 */
400 #endif /* __lint */
401
402 /*
403 * return the current frame pointer
404 */
405
406 #if defined(__lint)
407
408 greg_t
409 getfp(void)
410 { return (0); }
411
412 #else /* __lint */
413
414 #if defined(__amd64)
415
416 ENTRY(getfp)
417 movq %rbp, %rax
418 ret
419 SET_SIZE(getfp)
420
421 #elif defined(__i386)
422
423 ENTRY(getfp)
424 movl %ebp, %eax
425 ret
426 SET_SIZE(getfp)
427
428 #endif /* __i386 */
429 #endif /* __lint */
430
431 /*
432 * Invalidate a single page table entry in the TLB
433 */
434
435 #if defined(__lint)
436
437 /* ARGSUSED */
438 void
439 mmu_invlpg(caddr_t m)
440 {}
441
442 #else /* __lint */
443
444 ENTRY(mmu_invlpg)
445 invlpg (%rdi)
446 ret
447 SET_SIZE(mmu_invlpg)
448
449 #endif /* __lint */
450
451
452 /*
453 * Get/Set the value of various control registers
454 */
455
456 #if defined(__lint)
457
458 ulong_t
459 getcr0(void)
460 { return (0); }
461
462 /* ARGSUSED */
463 void
464 setcr0(ulong_t value)
465 {}
466
467 ulong_t
468 getcr2(void)
469 { return (0); }
470
471 ulong_t
472 getcr3(void)
473 { return (0); }
474
475 #if !defined(__xpv)
476 /* ARGSUSED */
477 void
478 setcr3(ulong_t val)
479 {}
480
481 void
482 reload_cr3(void)
483 {}
484 #endif
485
486 ulong_t
487 getcr4(void)
488 { return (0); }
489
490 /* ARGSUSED */
491 void
492 setcr4(ulong_t val)
493 {}
494
495 #if defined(__amd64)
496
497 ulong_t
498 getcr8(void)
499 { return (0); }
500
501 /* ARGSUSED */
502 void
503 setcr8(ulong_t val)
504 {}
505
506 #endif /* __amd64 */
507
508 #else /* __lint */
509
510 #if defined(__amd64)
511
512 ENTRY(getcr0)
513 movq %cr0, %rax
514 ret
515 SET_SIZE(getcr0)
516
517 ENTRY(setcr0)
518 movq %rdi, %cr0
519 ret
520 SET_SIZE(setcr0)
521
522 ENTRY(getcr2)
523 #if defined(__xpv)
524 movq %gs:CPU_VCPU_INFO, %rax
525 movq VCPU_INFO_ARCH_CR2(%rax), %rax
526 #else
527 movq %cr2, %rax
528 #endif
529 ret
530 SET_SIZE(getcr2)
531
532 ENTRY(getcr3)
533 movq %cr3, %rax
534 ret
535 SET_SIZE(getcr3)
536
537 #if !defined(__xpv)
538
539 ENTRY(setcr3)
540 movq %rdi, %cr3
541 ret
542 SET_SIZE(setcr3)
543
544 ENTRY(reload_cr3)
545 movq %cr3, %rdi
546 movq %rdi, %cr3
547 ret
548 SET_SIZE(reload_cr3)
549
550 #endif /* __xpv */
551
552 ENTRY(getcr4)
553 movq %cr4, %rax
554 ret
555 SET_SIZE(getcr4)
556
557 ENTRY(setcr4)
558 movq %rdi, %cr4
559 ret
560 SET_SIZE(setcr4)
561
562 ENTRY(getcr8)
563 movq %cr8, %rax
564 ret
565 SET_SIZE(getcr8)
566
567 ENTRY(setcr8)
568 movq %rdi, %cr8
569 ret
570 SET_SIZE(setcr8)
571
572 #elif defined(__i386)
573
574 ENTRY(getcr0)
575 movl %cr0, %eax
576 ret
577 SET_SIZE(getcr0)
578
579 ENTRY(setcr0)
580 movl 4(%esp), %eax
581 movl %eax, %cr0
582 ret
583 SET_SIZE(setcr0)
584
585 /*
586 * "lock mov %cr0" is used on processors which indicate it is
587 * supported via CPUID. Normally the 32 bit TPR is accessed via
588 * the local APIC.
589 */
590 ENTRY(getcr8)
591 lock
592 movl %cr0, %eax
593 ret
594 SET_SIZE(getcr8)
595
596 ENTRY(setcr8)
597 movl 4(%esp), %eax
598 lock
599 movl %eax, %cr0
600 ret
601 SET_SIZE(setcr8)
602
603 ENTRY(getcr2)
604 #if defined(__xpv)
605 movl %gs:CPU_VCPU_INFO, %eax
606 movl VCPU_INFO_ARCH_CR2(%eax), %eax
607 #else
608 movl %cr2, %eax
609 #endif
610 ret
611 SET_SIZE(getcr2)
612
613 ENTRY(getcr3)
614 movl %cr3, %eax
615 ret
616 SET_SIZE(getcr3)
617
618 #if !defined(__xpv)
619
620 ENTRY(setcr3)
621 movl 4(%esp), %eax
622 movl %eax, %cr3
623 ret
624 SET_SIZE(setcr3)
625
626 ENTRY(reload_cr3)
627 movl %cr3, %eax
628 movl %eax, %cr3
629 ret
630 SET_SIZE(reload_cr3)
631
632 #endif /* __xpv */
633
634 ENTRY(getcr4)
635 movl %cr4, %eax
636 ret
637 SET_SIZE(getcr4)
638
639 ENTRY(setcr4)
640 movl 4(%esp), %eax
641 movl %eax, %cr4
642 ret
643 SET_SIZE(setcr4)
644
645 #endif /* __i386 */
646 #endif /* __lint */
647
648 #if defined(__lint)
649
650 /*ARGSUSED*/
651 uint32_t
652 __cpuid_insn(struct cpuid_regs *regs)
653 { return (0); }
654
655 #else /* __lint */
656
657 #if defined(__amd64)
658
659 ENTRY(__cpuid_insn)
660 movq %rbx, %r8
661 movq %rcx, %r9
662 movq %rdx, %r11
663 movl (%rdi), %eax /* %eax = regs->cp_eax */
664 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
665 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
666 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
667 cpuid
668 movl %eax, (%rdi) /* regs->cp_eax = %eax */
669 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
670 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
671 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
672 movq %r8, %rbx
673 movq %r9, %rcx
674 movq %r11, %rdx
675 ret
676 SET_SIZE(__cpuid_insn)
677
678 #elif defined(__i386)
679
680 ENTRY(__cpuid_insn)
681 pushl %ebp
682 movl 0x8(%esp), %ebp /* %ebp = regs */
683 pushl %ebx
684 pushl %ecx
685 pushl %edx
686 movl (%ebp), %eax /* %eax = regs->cp_eax */
687 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
688 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
689 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
690 cpuid
691 movl %eax, (%ebp) /* regs->cp_eax = %eax */
692 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
693 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
694 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
695 popl %edx
696 popl %ecx
697 popl %ebx
698 popl %ebp
699 ret
700 SET_SIZE(__cpuid_insn)
701
702 #endif /* __i386 */
703 #endif /* __lint */
704
705 #if defined(__lint)
706
707 /*ARGSUSED*/
708 void
709 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
710 {}
711
712 #else /* __lint */
713
714 #if defined(__amd64)
715
716 ENTRY_NP(i86_monitor)
717 pushq %rbp
718 movq %rsp, %rbp
719 movq %rdi, %rax /* addr */
720 movq %rsi, %rcx /* extensions */
721 /* rdx contains input arg3: hints */
722 clflush (%rax)
723 .byte 0x0f, 0x01, 0xc8 /* monitor */
724 leave
725 ret
726 SET_SIZE(i86_monitor)
727
728 #elif defined(__i386)
729
730 ENTRY_NP(i86_monitor)
731 pushl %ebp
732 movl %esp, %ebp
733 movl 0x8(%ebp),%eax /* addr */
734 movl 0xc(%ebp),%ecx /* extensions */
735 movl 0x10(%ebp),%edx /* hints */
736 clflush (%eax)
737 .byte 0x0f, 0x01, 0xc8 /* monitor */
738 leave
739 ret
740 SET_SIZE(i86_monitor)
741
742 #endif /* __i386 */
743 #endif /* __lint */
744
745 #if defined(__lint)
746
747 /*ARGSUSED*/
748 void
749 i86_mwait(uint32_t data, uint32_t extensions)
750 {}
751
752 #else /* __lint */
753
754 #if defined(__amd64)
755
756 ENTRY_NP(i86_mwait)
757 pushq %rbp
758 call *x86_md_clear
759 movq %rsp, %rbp
760 movq %rdi, %rax /* data */
761 movq %rsi, %rcx /* extensions */
762 .byte 0x0f, 0x01, 0xc9 /* mwait */
763 leave
764 ret
765 SET_SIZE(i86_mwait)
766
767 #elif defined(__i386)
768
769 ENTRY_NP(i86_mwait)
770 pushl %ebp
771 movl %esp, %ebp
772 movl 0x8(%ebp),%eax /* data */
773 movl 0xc(%ebp),%ecx /* extensions */
774 .byte 0x0f, 0x01, 0xc9 /* mwait */
775 leave
776 ret
777 SET_SIZE(i86_mwait)
778
779 #endif /* __i386 */
780 #endif /* __lint */
781
782 #if defined(__xpv)
783 /*
784 * Defined in C
785 */
786 #else
787
788 #if defined(__lint)
789
790 hrtime_t
791 tsc_read(void)
792 {
793 return (0);
794 }
795
796 #else /* __lint */
797
798 #if defined(__amd64)
799
800 ENTRY_NP(tsc_read)
801 movq %rbx, %r11
802 movl $0, %eax
803 cpuid
804 rdtsc
805 movq %r11, %rbx
806 shlq $32, %rdx
807 orq %rdx, %rax
808 ret
809 .globl _tsc_mfence_start
810 _tsc_mfence_start:
811 mfence
812 rdtsc
813 shlq $32, %rdx
814 orq %rdx, %rax
815 ret
816 .globl _tsc_mfence_end
817 _tsc_mfence_end:
818 .globl _tscp_start
819 _tscp_start:
820 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
821 shlq $32, %rdx
822 orq %rdx, %rax
823 ret
824 .globl _tscp_end
825 _tscp_end:
826 .globl _no_rdtsc_start
827 _no_rdtsc_start:
828 xorl %edx, %edx
829 xorl %eax, %eax
830 ret
831 .globl _no_rdtsc_end
832 _no_rdtsc_end:
833 .globl _tsc_lfence_start
834 _tsc_lfence_start:
835 lfence
836 rdtsc
837 shlq $32, %rdx
838 orq %rdx, %rax
839 ret
840 .globl _tsc_lfence_end
841 _tsc_lfence_end:
842 SET_SIZE(tsc_read)
843
844 #else /* __i386 */
845
846 ENTRY_NP(tsc_read)
847 pushl %ebx
848 movl $0, %eax
849 cpuid
850 rdtsc
851 popl %ebx
852 ret
853 .globl _tsc_mfence_start
854 _tsc_mfence_start:
855 mfence
856 rdtsc
857 ret
858 .globl _tsc_mfence_end
859 _tsc_mfence_end:
860 .globl _tscp_start
861 _tscp_start:
862 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
863 ret
864 .globl _tscp_end
865 _tscp_end:
866 .globl _no_rdtsc_start
867 _no_rdtsc_start:
868 xorl %edx, %edx
869 xorl %eax, %eax
870 ret
871 .globl _no_rdtsc_end
872 _no_rdtsc_end:
873 .globl _tsc_lfence_start
874 _tsc_lfence_start:
875 lfence
876 rdtsc
877 ret
878 .globl _tsc_lfence_end
879 _tsc_lfence_end:
880 SET_SIZE(tsc_read)
881
882 #endif /* __i386 */
883
884 #endif /* __lint */
885
886
887 #endif /* __xpv */
888
889 #ifdef __lint
890 /*
891 * Do not use this function for obtaining clock tick. This
892 * is called by callers who do not need to have a guarenteed
893 * correct tick value. The proper routine to use is tsc_read().
894 */
895 u_longlong_t
896 randtick(void)
897 {
898 return (0);
899 }
900 #else
901 #if defined(__amd64)
902 ENTRY_NP(randtick)
903 rdtsc
904 shlq $32, %rdx
905 orq %rdx, %rax
906 ret
907 SET_SIZE(randtick)
908 #else
909 ENTRY_NP(randtick)
910 rdtsc
911 ret
912 SET_SIZE(randtick)
913 #endif /* __i386 */
914 #endif /* __lint */
915 /*
916 * Insert entryp after predp in a doubly linked list.
917 */
918
919 #if defined(__lint)
920
921 /*ARGSUSED*/
922 void
923 _insque(caddr_t entryp, caddr_t predp)
924 {}
925
926 #else /* __lint */
927
928 #if defined(__amd64)
929
930 ENTRY(_insque)
931 movq (%rsi), %rax /* predp->forw */
932 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
933 movq %rax, (%rdi) /* entryp->forw = predp->forw */
934 movq %rdi, (%rsi) /* predp->forw = entryp */
935 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
936 ret
937 SET_SIZE(_insque)
938
939 #elif defined(__i386)
940
941 ENTRY(_insque)
942 movl 8(%esp), %edx
943 movl 4(%esp), %ecx
944 movl (%edx), %eax /* predp->forw */
945 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
946 movl %eax, (%ecx) /* entryp->forw = predp->forw */
947 movl %ecx, (%edx) /* predp->forw = entryp */
948 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
949 ret
950 SET_SIZE(_insque)
951
952 #endif /* __i386 */
953 #endif /* __lint */
954
955 /*
956 * Remove entryp from a doubly linked list
957 */
958
959 #if defined(__lint)
960
961 /*ARGSUSED*/
962 void
963 _remque(caddr_t entryp)
964 {}
965
966 #else /* __lint */
967
968 #if defined(__amd64)
969
970 ENTRY(_remque)
971 movq (%rdi), %rax /* entry->forw */
972 movq CPTRSIZE(%rdi), %rdx /* entry->back */
973 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
974 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
975 ret
976 SET_SIZE(_remque)
977
978 #elif defined(__i386)
979
980 ENTRY(_remque)
981 movl 4(%esp), %ecx
982 movl (%ecx), %eax /* entry->forw */
983 movl CPTRSIZE(%ecx), %edx /* entry->back */
984 movl %eax, (%edx) /* entry->back->forw = entry->forw */
985 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
986 ret
987 SET_SIZE(_remque)
988
989 #endif /* __i386 */
990 #endif /* __lint */
991
992 /*
993 * Returns the number of
994 * non-NULL bytes in string argument.
995 */
996
997 #if defined(__lint)
998
999 /* ARGSUSED */
1000 size_t
1001 strlen(const char *str)
1002 { return (0); }
1003
1004 #else /* __lint */
1005
1006 #if defined(__amd64)
1007
1008 /*
1009 * This is close to a simple transliteration of a C version of this
1010 * routine. We should either just -make- this be a C version, or
1011 * justify having it in assembler by making it significantly faster.
1012 *
1013 * size_t
1014 * strlen(const char *s)
1015 * {
1016 * const char *s0;
1017 * #if defined(DEBUG)
1018 * if ((uintptr_t)s < KERNELBASE)
1019 * panic(.str_panic_msg);
1020 * #endif
1021 * for (s0 = s; *s; s++)
1022 * ;
1023 * return (s - s0);
1024 * }
1025 */
1026
1027 ENTRY(strlen)
1028 #ifdef DEBUG
1029 movq postbootkernelbase(%rip), %rax
1030 cmpq %rax, %rdi
1031 jae str_valid
1032 pushq %rbp
1033 movq %rsp, %rbp
1034 leaq .str_panic_msg(%rip), %rdi
1035 xorl %eax, %eax
1036 call panic
1037 #endif /* DEBUG */
1038 str_valid:
1039 cmpb $0, (%rdi)
1040 movq %rdi, %rax
1041 je .null_found
1042 .align 4
1043 .strlen_loop:
1044 incq %rdi
1045 cmpb $0, (%rdi)
1046 jne .strlen_loop
1047 .null_found:
1048 subq %rax, %rdi
1049 movq %rdi, %rax
1050 ret
1051 SET_SIZE(strlen)
1052
1053 #elif defined(__i386)
1054
1055 ENTRY(strlen)
1056 #ifdef DEBUG
1057 movl postbootkernelbase, %eax
1058 cmpl %eax, 4(%esp)
1059 jae str_valid
1060 pushl %ebp
1061 movl %esp, %ebp
1062 pushl $.str_panic_msg
1063 call panic
1064 #endif /* DEBUG */
1065
1066 str_valid:
1067 movl 4(%esp), %eax /* %eax = string address */
1068 testl $3, %eax /* if %eax not word aligned */
1069 jnz .not_word_aligned /* goto .not_word_aligned */
1070 .align 4
1071 .word_aligned:
1072 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1073 movl $0x7f7f7f7f, %ecx
1074 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1075 addl $4, %eax /* next word */
1076 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1077 orl %edx, %ecx /* %ecx |= %edx */
1078 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1079 cmpl $0x80808080, %ecx /* if no null byte in this word */
1080 je .word_aligned /* goto .word_aligned */
1081 subl $4, %eax /* post-incremented */
1082 .not_word_aligned:
1083 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1084 je .null_found /* goto .null_found */
1085 incl %eax /* next byte */
1086 testl $3, %eax /* if %eax not word aligned */
1087 jnz .not_word_aligned /* goto .not_word_aligned */
1088 jmp .word_aligned /* goto .word_aligned */
1089 .align 4
1090 .null_found:
1091 subl 4(%esp), %eax /* %eax -= string address */
1092 ret
1093 SET_SIZE(strlen)
1094
1095 #endif /* __i386 */
1096
1097 #ifdef DEBUG
1098 .text
1099 .str_panic_msg:
1100 .string "strlen: argument below kernelbase"
1101 #endif /* DEBUG */
1102
1103 #endif /* __lint */
1104
1105 /*
1106 * Berkeley 4.3 introduced symbolically named interrupt levels
1107 * as a way deal with priority in a machine independent fashion.
1108 * Numbered priorities are machine specific, and should be
1109 * discouraged where possible.
1110 *
1111 * Note, for the machine specific priorities there are
1112 * examples listed for devices that use a particular priority.
1113 * It should not be construed that all devices of that
1114 * type should be at that priority. It is currently were
1115 * the current devices fit into the priority scheme based
1116 * upon time criticalness.
1117 *
1118 * The underlying assumption of these assignments is that
1119 * IPL 10 is the highest level from which a device
1120 * routine can call wakeup. Devices that interrupt from higher
1121 * levels are restricted in what they can do. If they need
1122 * kernels services they should schedule a routine at a lower
1123 * level (via software interrupt) to do the required
1124 * processing.
1125 *
1126 * Examples of this higher usage:
1127 * Level Usage
1128 * 14 Profiling clock (and PROM uart polling clock)
1129 * 12 Serial ports
1130 *
1131 * The serial ports request lower level processing on level 6.
1132 *
1133 * Also, almost all splN routines (where N is a number or a
1134 * mnemonic) will do a RAISE(), on the assumption that they are
1135 * never used to lower our priority.
1136 * The exceptions are:
1137 * spl8() Because you can't be above 15 to begin with!
1138 * splzs() Because this is used at boot time to lower our
1139 * priority, to allow the PROM to poll the uart.
1140 * spl0() Used to lower priority to 0.
1141 */
1142
1143 #if defined(__lint)
1144
1145 int spl0(void) { return (0); }
1146 int spl6(void) { return (0); }
1147 int spl7(void) { return (0); }
1148 int spl8(void) { return (0); }
1149 int splhigh(void) { return (0); }
1150 int splhi(void) { return (0); }
1151 int splzs(void) { return (0); }
1152
1153 /* ARGSUSED */
1154 void
1155 splx(int level)
1156 {}
1157
1158 #else /* __lint */
1159
1160 #if defined(__amd64)
1161
1162 #define SETPRI(level) \
1163 movl $/**/level, %edi; /* new priority */ \
1164 jmp do_splx /* redirect to do_splx */
1165
1166 #define RAISE(level) \
1167 movl $/**/level, %edi; /* new priority */ \
1168 jmp splr /* redirect to splr */
1169
1170 #elif defined(__i386)
1171
1172 #define SETPRI(level) \
1173 pushl $/**/level; /* new priority */ \
1174 call do_splx; /* invoke common splx code */ \
1175 addl $4, %esp; /* unstack arg */ \
1176 ret
1177
1178 #define RAISE(level) \
1179 pushl $/**/level; /* new priority */ \
1180 call splr; /* invoke common splr code */ \
1181 addl $4, %esp; /* unstack args */ \
1182 ret
1183
1184 #endif /* __i386 */
1185
1186 /* locks out all interrupts, including memory errors */
1187 ENTRY(spl8)
1188 SETPRI(15)
1189 SET_SIZE(spl8)
1190
1191 /* just below the level that profiling runs */
1192 ENTRY(spl7)
1193 RAISE(13)
1194 SET_SIZE(spl7)
1195
1196 /* sun specific - highest priority onboard serial i/o asy ports */
1197 ENTRY(splzs)
1198 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1199 SET_SIZE(splzs)
1200
1201 ENTRY(splhi)
1202 ALTENTRY(splhigh)
1203 ALTENTRY(spl6)
1204 ALTENTRY(i_ddi_splhigh)
1205
1206 RAISE(DISP_LEVEL)
1207
1208 SET_SIZE(i_ddi_splhigh)
1209 SET_SIZE(spl6)
1210 SET_SIZE(splhigh)
1211 SET_SIZE(splhi)
1212
1213 /* allow all interrupts */
1214 ENTRY(spl0)
1215 SETPRI(0)
1216 SET_SIZE(spl0)
1217
1218
1219 /* splx implementation */
1220 ENTRY(splx)
1221 jmp do_splx /* redirect to common splx code */
1222 SET_SIZE(splx)
1223
1224 #endif /* __lint */
1225
1226 #if defined(__i386)
1227
1228 /*
1229 * Read and write the %gs register
1230 */
1231
1232 #if defined(__lint)
1233
1234 /*ARGSUSED*/
1235 uint16_t
1236 getgs(void)
1237 { return (0); }
1238
1239 /*ARGSUSED*/
1240 void
1241 setgs(uint16_t sel)
1242 {}
1243
1244 #else /* __lint */
1245
1246 ENTRY(getgs)
1247 clr %eax
1248 movw %gs, %ax
1249 ret
1250 SET_SIZE(getgs)
1251
1252 ENTRY(setgs)
1253 movw 4(%esp), %gs
1254 ret
1255 SET_SIZE(setgs)
1256
1257 #endif /* __lint */
1258 #endif /* __i386 */
1259
1260 #if defined(__lint)
1261
1262 void
1263 pc_reset(void)
1264 {}
1265
1266 void
1267 efi_reset(void)
1268 {}
1269
1270 #else /* __lint */
1271
1272 ENTRY(wait_500ms)
1273 #if defined(__amd64)
1274 pushq %rbx
1275 #elif defined(__i386)
1276 push %ebx
1277 #endif
1278 movl $50000, %ebx
1279 1:
1280 call tenmicrosec
1281 decl %ebx
1282 jnz 1b
1283 #if defined(__amd64)
1284 popq %rbx
1285 #elif defined(__i386)
1286 pop %ebx
1287 #endif
1288 ret
1289 SET_SIZE(wait_500ms)
1290
1291 #define RESET_METHOD_KBC 1
1292 #define RESET_METHOD_PORT92 2
1293 #define RESET_METHOD_PCI 4
1294
1295 DGDEF3(pc_reset_methods, 4, 8)
1296 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1297
1298 ENTRY(pc_reset)
1299
1300 #if defined(__i386)
1301 testl $RESET_METHOD_KBC, pc_reset_methods
1302 #elif defined(__amd64)
1303 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1304 #endif
1305 jz 1f
1306
1307 /
1308 / Try the classic keyboard controller-triggered reset.
1309 /
1310 movw $0x64, %dx
1311 movb $0xfe, %al
1312 outb (%dx)
1313
1314 / Wait up to 500 milliseconds here for the keyboard controller
1315 / to pull the reset line. On some systems where the keyboard
1316 / controller is slow to pull the reset line, the next reset method
1317 / may be executed (which may be bad if those systems hang when the
1318 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1319 / and Ferrari 4000 (doesn't like the cf9 reset method))
1320
1321 call wait_500ms
1322
1323 1:
1324 #if defined(__i386)
1325 testl $RESET_METHOD_PORT92, pc_reset_methods
1326 #elif defined(__amd64)
1327 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1328 #endif
1329 jz 3f
1330
1331 /
1332 / Try port 0x92 fast reset
1333 /
1334 movw $0x92, %dx
1335 inb (%dx)
1336 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1337 je 1f
1338 testb $1, %al / If bit 0
1339 jz 2f / is clear, jump to perform the reset
1340 andb $0xfe, %al / otherwise,
1341 outb (%dx) / clear bit 0 first, then
1342 2:
1343 orb $1, %al / Set bit 0
1344 outb (%dx) / and reset the system
1345 1:
1346
1347 call wait_500ms
1348
1349 3:
1350 #if defined(__i386)
1351 testl $RESET_METHOD_PCI, pc_reset_methods
1352 #elif defined(__amd64)
1353 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1354 #endif
1355 jz 4f
1356
1357 / Try the PCI (soft) reset vector (should work on all modern systems,
1358 / but has been shown to cause problems on 450NX systems, and some newer
1359 / systems (e.g. ATI IXP400-equipped systems))
1360 / When resetting via this method, 2 writes are required. The first
1361 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1362 / power cycle).
1363 / The reset occurs on the second write, during bit 2's transition from
1364 / 0->1.
1365 movw $0xcf9, %dx
1366 movb $0x2, %al / Reset mode = hard, no power cycle
1367 outb (%dx)
1368 movb $0x6, %al
1369 outb (%dx)
1370
1371 call wait_500ms
1372
1373 4:
1374 /
1375 / port 0xcf9 failed also. Last-ditch effort is to
1376 / triple-fault the CPU.
1377 / Also, use triple fault for EFI firmware
1378 /
1379 ENTRY(efi_reset)
1380 #if defined(__amd64)
1381 pushq $0x0
1382 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1383 lidt (%rsp)
1384 #elif defined(__i386)
1385 pushl $0x0
1386 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1387 lidt (%esp)
1388 #endif
1389 int $0x0 / Trigger interrupt, generate triple-fault
1390
1391 cli
1392 hlt / Wait forever
1393 /*NOTREACHED*/
1394 SET_SIZE(efi_reset)
1395 SET_SIZE(pc_reset)
1396
1397 #endif /* __lint */
1398
1399 /*
1400 * C callable in and out routines
1401 */
1402
1403 #if defined(__lint)
1404
1405 /* ARGSUSED */
1406 void
1407 outl(int port_address, uint32_t val)
1408 {}
1409
1410 #else /* __lint */
1411
1412 #if defined(__amd64)
1413
1414 ENTRY(outl)
1415 movw %di, %dx
1416 movl %esi, %eax
1417 outl (%dx)
1418 ret
1419 SET_SIZE(outl)
1420
1421 #elif defined(__i386)
1422
1423 .set PORT, 4
1424 .set VAL, 8
1425
1426 ENTRY(outl)
1427 movw PORT(%esp), %dx
1428 movl VAL(%esp), %eax
1429 outl (%dx)
1430 ret
1431 SET_SIZE(outl)
1432
1433 #endif /* __i386 */
1434 #endif /* __lint */
1435
1436 #if defined(__lint)
1437
1438 /* ARGSUSED */
1439 void
1440 outw(int port_address, uint16_t val)
1441 {}
1442
1443 #else /* __lint */
1444
1445 #if defined(__amd64)
1446
1447 ENTRY(outw)
1448 movw %di, %dx
1449 movw %si, %ax
1450 D16 outl (%dx) /* XX64 why not outw? */
1451 ret
1452 SET_SIZE(outw)
1453
1454 #elif defined(__i386)
1455
1456 ENTRY(outw)
1457 movw PORT(%esp), %dx
1458 movw VAL(%esp), %ax
1459 D16 outl (%dx)
1460 ret
1461 SET_SIZE(outw)
1462
1463 #endif /* __i386 */
1464 #endif /* __lint */
1465
1466 #if defined(__lint)
1467
1468 /* ARGSUSED */
1469 void
1470 outb(int port_address, uint8_t val)
1471 {}
1472
1473 #else /* __lint */
1474
1475 #if defined(__amd64)
1476
1477 ENTRY(outb)
1478 movw %di, %dx
1479 movb %sil, %al
1480 outb (%dx)
1481 ret
1482 SET_SIZE(outb)
1483
1484 #elif defined(__i386)
1485
1486 ENTRY(outb)
1487 movw PORT(%esp), %dx
1488 movb VAL(%esp), %al
1489 outb (%dx)
1490 ret
1491 SET_SIZE(outb)
1492
1493 #endif /* __i386 */
1494 #endif /* __lint */
1495
1496 #if defined(__lint)
1497
1498 /* ARGSUSED */
1499 uint32_t
1500 inl(int port_address)
1501 { return (0); }
1502
1503 #else /* __lint */
1504
1505 #if defined(__amd64)
1506
1507 ENTRY(inl)
1508 xorl %eax, %eax
1509 movw %di, %dx
1510 inl (%dx)
1511 ret
1512 SET_SIZE(inl)
1513
1514 #elif defined(__i386)
1515
1516 ENTRY(inl)
1517 movw PORT(%esp), %dx
1518 inl (%dx)
1519 ret
1520 SET_SIZE(inl)
1521
1522 #endif /* __i386 */
1523 #endif /* __lint */
1524
1525 #if defined(__lint)
1526
1527 /* ARGSUSED */
1528 uint16_t
1529 inw(int port_address)
1530 { return (0); }
1531
1532 #else /* __lint */
1533
1534 #if defined(__amd64)
1535
1536 ENTRY(inw)
1537 xorl %eax, %eax
1538 movw %di, %dx
1539 D16 inl (%dx)
1540 ret
1541 SET_SIZE(inw)
1542
1543 #elif defined(__i386)
1544
1545 ENTRY(inw)
1546 subl %eax, %eax
1547 movw PORT(%esp), %dx
1548 D16 inl (%dx)
1549 ret
1550 SET_SIZE(inw)
1551
1552 #endif /* __i386 */
1553 #endif /* __lint */
1554
1555
1556 #if defined(__lint)
1557
1558 /* ARGSUSED */
1559 uint8_t
1560 inb(int port_address)
1561 { return (0); }
1562
1563 #else /* __lint */
1564
1565 #if defined(__amd64)
1566
1567 ENTRY(inb)
1568 xorl %eax, %eax
1569 movw %di, %dx
1570 inb (%dx)
1571 ret
1572 SET_SIZE(inb)
1573
1574 #elif defined(__i386)
1575
1576 ENTRY(inb)
1577 subl %eax, %eax
1578 movw PORT(%esp), %dx
1579 inb (%dx)
1580 ret
1581 SET_SIZE(inb)
1582
1583 #endif /* __i386 */
1584 #endif /* __lint */
1585
1586
1587 #if defined(__lint)
1588
1589 /* ARGSUSED */
1590 void
1591 repoutsw(int port, uint16_t *addr, int cnt)
1592 {}
1593
1594 #else /* __lint */
1595
1596 #if defined(__amd64)
1597
1598 ENTRY(repoutsw)
1599 movl %edx, %ecx
1600 movw %di, %dx
1601 rep
1602 D16 outsl
1603 ret
1604 SET_SIZE(repoutsw)
1605
1606 #elif defined(__i386)
1607
1608 /*
1609 * The arguments and saved registers are on the stack in the
1610 * following order:
1611 * | cnt | +16
1612 * | *addr | +12
1613 * | port | +8
1614 * | eip | +4
1615 * | esi | <-- %esp
1616 * If additional values are pushed onto the stack, make sure
1617 * to adjust the following constants accordingly.
1618 */
1619 .set PORT, 8
1620 .set ADDR, 12
1621 .set COUNT, 16
1622
1623 ENTRY(repoutsw)
1624 pushl %esi
1625 movl PORT(%esp), %edx
1626 movl ADDR(%esp), %esi
1627 movl COUNT(%esp), %ecx
1628 rep
1629 D16 outsl
1630 popl %esi
1631 ret
1632 SET_SIZE(repoutsw)
1633
1634 #endif /* __i386 */
1635 #endif /* __lint */
1636
1637
1638 #if defined(__lint)
1639
1640 /* ARGSUSED */
1641 void
1642 repinsw(int port_addr, uint16_t *addr, int cnt)
1643 {}
1644
1645 #else /* __lint */
1646
1647 #if defined(__amd64)
1648
1649 ENTRY(repinsw)
1650 movl %edx, %ecx
1651 movw %di, %dx
1652 rep
1653 D16 insl
1654 ret
1655 SET_SIZE(repinsw)
1656
1657 #elif defined(__i386)
1658
1659 ENTRY(repinsw)
1660 pushl %edi
1661 movl PORT(%esp), %edx
1662 movl ADDR(%esp), %edi
1663 movl COUNT(%esp), %ecx
1664 rep
1665 D16 insl
1666 popl %edi
1667 ret
1668 SET_SIZE(repinsw)
1669
1670 #endif /* __i386 */
1671 #endif /* __lint */
1672
1673
1674 #if defined(__lint)
1675
1676 /* ARGSUSED */
1677 void
1678 repinsb(int port, uint8_t *addr, int count)
1679 {}
1680
1681 #else /* __lint */
1682
1683 #if defined(__amd64)
1684
1685 ENTRY(repinsb)
1686 movl %edx, %ecx
1687 movw %di, %dx
1688 movq %rsi, %rdi
1689 rep
1690 insb
1691 ret
1692 SET_SIZE(repinsb)
1693
1694 #elif defined(__i386)
1695
1696 /*
1697 * The arguments and saved registers are on the stack in the
1698 * following order:
1699 * | cnt | +16
1700 * | *addr | +12
1701 * | port | +8
1702 * | eip | +4
1703 * | esi | <-- %esp
1704 * If additional values are pushed onto the stack, make sure
1705 * to adjust the following constants accordingly.
1706 */
1707 .set IO_PORT, 8
1708 .set IO_ADDR, 12
1709 .set IO_COUNT, 16
1710
1711 ENTRY(repinsb)
1712 pushl %edi
1713 movl IO_ADDR(%esp), %edi
1714 movl IO_COUNT(%esp), %ecx
1715 movl IO_PORT(%esp), %edx
1716 rep
1717 insb
1718 popl %edi
1719 ret
1720 SET_SIZE(repinsb)
1721
1722 #endif /* __i386 */
1723 #endif /* __lint */
1724
1725
1726 /*
1727 * Input a stream of 32-bit words.
1728 * NOTE: count is a DWORD count.
1729 */
1730 #if defined(__lint)
1731
1732 /* ARGSUSED */
1733 void
1734 repinsd(int port, uint32_t *addr, int count)
1735 {}
1736
1737 #else /* __lint */
1738
1739 #if defined(__amd64)
1740
1741 ENTRY(repinsd)
1742 movl %edx, %ecx
1743 movw %di, %dx
1744 movq %rsi, %rdi
1745 rep
1746 insl
1747 ret
1748 SET_SIZE(repinsd)
1749
1750 #elif defined(__i386)
1751
1752 ENTRY(repinsd)
1753 pushl %edi
1754 movl IO_ADDR(%esp), %edi
1755 movl IO_COUNT(%esp), %ecx
1756 movl IO_PORT(%esp), %edx
1757 rep
1758 insl
1759 popl %edi
1760 ret
1761 SET_SIZE(repinsd)
1762
1763 #endif /* __i386 */
1764 #endif /* __lint */
1765
1766 /*
1767 * Output a stream of bytes
1768 * NOTE: count is a byte count
1769 */
1770 #if defined(__lint)
1771
1772 /* ARGSUSED */
1773 void
1774 repoutsb(int port, uint8_t *addr, int count)
1775 {}
1776
1777 #else /* __lint */
1778
1779 #if defined(__amd64)
1780
1781 ENTRY(repoutsb)
1782 movl %edx, %ecx
1783 movw %di, %dx
1784 rep
1785 outsb
1786 ret
1787 SET_SIZE(repoutsb)
1788
1789 #elif defined(__i386)
1790
1791 ENTRY(repoutsb)
1792 pushl %esi
1793 movl IO_ADDR(%esp), %esi
1794 movl IO_COUNT(%esp), %ecx
1795 movl IO_PORT(%esp), %edx
1796 rep
1797 outsb
1798 popl %esi
1799 ret
1800 SET_SIZE(repoutsb)
1801
1802 #endif /* __i386 */
1803 #endif /* __lint */
1804
1805 /*
1806 * Output a stream of 32-bit words
1807 * NOTE: count is a DWORD count
1808 */
1809 #if defined(__lint)
1810
1811 /* ARGSUSED */
1812 void
1813 repoutsd(int port, uint32_t *addr, int count)
1814 {}
1815
1816 #else /* __lint */
1817
1818 #if defined(__amd64)
1819
1820 ENTRY(repoutsd)
1821 movl %edx, %ecx
1822 movw %di, %dx
1823 rep
1824 outsl
1825 ret
1826 SET_SIZE(repoutsd)
1827
1828 #elif defined(__i386)
1829
1830 ENTRY(repoutsd)
1831 pushl %esi
1832 movl IO_ADDR(%esp), %esi
1833 movl IO_COUNT(%esp), %ecx
1834 movl IO_PORT(%esp), %edx
1835 rep
1836 outsl
1837 popl %esi
1838 ret
1839 SET_SIZE(repoutsd)
1840
1841 #endif /* __i386 */
1842 #endif /* __lint */
1843
1844 /*
1845 * void int3(void)
1846 * void int18(void)
1847 * void int20(void)
1848 * void int_cmci(void)
1849 */
1850
1851 #if defined(__lint)
1852
1853 void
1854 int3(void)
1855 {}
1856
1857 void
1858 int18(void)
1859 {}
1860
1861 void
1862 int20(void)
1863 {}
1864
1865 void
1866 int_cmci(void)
1867 {}
1868
1869 #else /* __lint */
1870
1871 ENTRY(int3)
1872 int $T_BPTFLT
1873 ret
1874 SET_SIZE(int3)
1875
1876 ENTRY(int18)
1877 int $T_MCE
1878 ret
1879 SET_SIZE(int18)
1880
1881 ENTRY(int20)
1882 movl boothowto, %eax
1883 andl $RB_DEBUG, %eax
1884 jz 1f
1885
1886 int $T_DBGENTR
1887 1:
1888 rep; ret /* use 2 byte return instruction when branch target */
1889 /* AMD Software Optimization Guide - Section 6.2 */
1890 SET_SIZE(int20)
1891
1892 ENTRY(int_cmci)
1893 int $T_ENOEXTFLT
1894 ret
1895 SET_SIZE(int_cmci)
1896
1897 #endif /* __lint */
1898
1899 #if defined(__lint)
1900
1901 /* ARGSUSED */
1902 int
1903 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1904 { return (0); }
1905
1906 #else /* __lint */
1907
1908 #if defined(__amd64)
1909
1910 ENTRY(scanc)
1911 /* rdi == size */
1912 /* rsi == cp */
1913 /* rdx == table */
1914 /* rcx == mask */
1915 addq %rsi, %rdi /* end = &cp[size] */
1916 .scanloop:
1917 cmpq %rdi, %rsi /* while (cp < end */
1918 jnb .scandone
1919 movzbq (%rsi), %r8 /* %r8 = *cp */
1920 incq %rsi /* cp++ */
1921 testb %cl, (%r8, %rdx)
1922 jz .scanloop /* && (table[*cp] & mask) == 0) */
1923 decq %rsi /* (fix post-increment) */
1924 .scandone:
1925 movl %edi, %eax
1926 subl %esi, %eax /* return (end - cp) */
1927 ret
1928 SET_SIZE(scanc)
1929
1930 #elif defined(__i386)
1931
1932 ENTRY(scanc)
1933 pushl %edi
1934 pushl %esi
1935 movb 24(%esp), %cl /* mask = %cl */
1936 movl 16(%esp), %esi /* cp = %esi */
1937 movl 20(%esp), %edx /* table = %edx */
1938 movl %esi, %edi
1939 addl 12(%esp), %edi /* end = &cp[size]; */
1940 .scanloop:
1941 cmpl %edi, %esi /* while (cp < end */
1942 jnb .scandone
1943 movzbl (%esi), %eax /* %al = *cp */
1944 incl %esi /* cp++ */
1945 movb (%edx, %eax), %al /* %al = table[*cp] */
1946 testb %al, %cl
1947 jz .scanloop /* && (table[*cp] & mask) == 0) */
1948 dec %esi /* post-incremented */
1949 .scandone:
1950 movl %edi, %eax
1951 subl %esi, %eax /* return (end - cp) */
1952 popl %esi
1953 popl %edi
1954 ret
1955 SET_SIZE(scanc)
1956
1957 #endif /* __i386 */
1958 #endif /* __lint */
1959
1960 /*
1961 * Replacement functions for ones that are normally inlined.
1962 * In addition to the copy in i86.il, they are defined here just in case.
1963 */
1964
1965 #if defined(__lint)
1966
1967 ulong_t
1968 intr_clear(void)
1969 { return (0); }
1970
1971 ulong_t
1972 clear_int_flag(void)
1973 { return (0); }
1974
1975 #else /* __lint */
1976
1977 #if defined(__amd64)
1978
1979 ENTRY(intr_clear)
1980 ENTRY(clear_int_flag)
1981 pushfq
1982 popq %rax
1983 #if defined(__xpv)
1984 leaq xpv_panicking, %rdi
1985 movl (%rdi), %edi
1986 cmpl $0, %edi
1987 jne 2f
1988 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1989 /*
1990 * Synthesize the PS_IE bit from the event mask bit
1991 */
1992 andq $_BITNOT(PS_IE), %rax
1993 testb $1, %dl
1994 jnz 1f
1995 orq $PS_IE, %rax
1996 1:
1997 ret
1998 2:
1999 #endif
2000 CLI(%rdi)
2001 ret
2002 SET_SIZE(clear_int_flag)
2003 SET_SIZE(intr_clear)
2004
2005 #elif defined(__i386)
2006
2007 ENTRY(intr_clear)
2008 ENTRY(clear_int_flag)
2009 pushfl
2010 popl %eax
2011 #if defined(__xpv)
2012 leal xpv_panicking, %edx
2013 movl (%edx), %edx
2014 cmpl $0, %edx
2015 jne 2f
2016 CLIRET(%edx, %cl) /* returns event mask in %cl */
2017 /*
2018 * Synthesize the PS_IE bit from the event mask bit
2019 */
2020 andl $_BITNOT(PS_IE), %eax
2021 testb $1, %cl
2022 jnz 1f
2023 orl $PS_IE, %eax
2024 1:
2025 ret
2026 2:
2027 #endif
2028 CLI(%edx)
2029 ret
2030 SET_SIZE(clear_int_flag)
2031 SET_SIZE(intr_clear)
2032
2033 #endif /* __i386 */
2034 #endif /* __lint */
2035
2036 #if defined(__lint)
2037
2038 struct cpu *
2039 curcpup(void)
2040 { return 0; }
2041
2042 #else /* __lint */
2043
2044 #if defined(__amd64)
2045
2046 ENTRY(curcpup)
2047 movq %gs:CPU_SELF, %rax
2048 ret
2049 SET_SIZE(curcpup)
2050
2051 #elif defined(__i386)
2052
2053 ENTRY(curcpup)
2054 movl %gs:CPU_SELF, %eax
2055 ret
2056 SET_SIZE(curcpup)
2057
2058 #endif /* __i386 */
2059 #endif /* __lint */
2060
2061 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2062 * These functions reverse the byte order of the input parameter and returns
2063 * the result. This is to convert the byte order from host byte order
2064 * (little endian) to network byte order (big endian), or vice versa.
2065 */
2066
2067 #if defined(__lint)
2068
2069 uint64_t
2070 htonll(uint64_t i)
2071 { return (i); }
2072
2073 uint64_t
2074 ntohll(uint64_t i)
2075 { return (i); }
2076
2077 uint32_t
2078 htonl(uint32_t i)
2079 { return (i); }
2080
2081 uint32_t
2082 ntohl(uint32_t i)
2083 { return (i); }
2084
2085 uint16_t
2086 htons(uint16_t i)
2087 { return (i); }
2088
2089 uint16_t
2090 ntohs(uint16_t i)
2091 { return (i); }
2092
2093 #else /* __lint */
2094
2095 #if defined(__amd64)
2096
2097 ENTRY(htonll)
2098 ALTENTRY(ntohll)
2099 movq %rdi, %rax
2100 bswapq %rax
2101 ret
2102 SET_SIZE(ntohll)
2103 SET_SIZE(htonll)
2104
2105 /* XX64 there must be shorter sequences for this */
2106 ENTRY(htonl)
2107 ALTENTRY(ntohl)
2108 movl %edi, %eax
2109 bswap %eax
2110 ret
2111 SET_SIZE(ntohl)
2112 SET_SIZE(htonl)
2113
2114 /* XX64 there must be better sequences for this */
2115 ENTRY(htons)
2116 ALTENTRY(ntohs)
2117 movl %edi, %eax
2118 bswap %eax
2119 shrl $16, %eax
2120 ret
2121 SET_SIZE(ntohs)
2122 SET_SIZE(htons)
2123
2124 #elif defined(__i386)
2125
2126 ENTRY(htonll)
2127 ALTENTRY(ntohll)
2128 movl 4(%esp), %edx
2129 movl 8(%esp), %eax
2130 bswap %edx
2131 bswap %eax
2132 ret
2133 SET_SIZE(ntohll)
2134 SET_SIZE(htonll)
2135
2136 ENTRY(htonl)
2137 ALTENTRY(ntohl)
2138 movl 4(%esp), %eax
2139 bswap %eax
2140 ret
2141 SET_SIZE(ntohl)
2142 SET_SIZE(htonl)
2143
2144 ENTRY(htons)
2145 ALTENTRY(ntohs)
2146 movl 4(%esp), %eax
2147 bswap %eax
2148 shrl $16, %eax
2149 ret
2150 SET_SIZE(ntohs)
2151 SET_SIZE(htons)
2152
2153 #endif /* __i386 */
2154 #endif /* __lint */
2155
2156
2157 #if defined(__lint)
2158
2159 /* ARGSUSED */
2160 void
2161 intr_restore(ulong_t i)
2162 { return; }
2163
2164 /* ARGSUSED */
2165 void
2166 restore_int_flag(ulong_t i)
2167 { return; }
2168
2169 #else /* __lint */
2170
2171 #if defined(__amd64)
2172
2173 ENTRY(intr_restore)
2174 ENTRY(restore_int_flag)
2175 testq $PS_IE, %rdi
2176 jz 1f
2177 #if defined(__xpv)
2178 leaq xpv_panicking, %rsi
2179 movl (%rsi), %esi
2180 cmpl $0, %esi
2181 jne 1f
2182 /*
2183 * Since we're -really- running unprivileged, our attempt
2184 * to change the state of the IF bit will be ignored.
2185 * The virtual IF bit is tweaked by CLI and STI.
2186 */
2187 IE_TO_EVENT_MASK(%rsi, %rdi)
2188 #else
2189 sti
2190 #endif
2191 1:
2192 ret
2193 SET_SIZE(restore_int_flag)
2194 SET_SIZE(intr_restore)
2195
2196 #elif defined(__i386)
2197
2198 ENTRY(intr_restore)
2199 ENTRY(restore_int_flag)
2200 testl $PS_IE, 4(%esp)
2201 jz 1f
2202 #if defined(__xpv)
2203 leal xpv_panicking, %edx
2204 movl (%edx), %edx
2205 cmpl $0, %edx
2206 jne 1f
2207 /*
2208 * Since we're -really- running unprivileged, our attempt
2209 * to change the state of the IF bit will be ignored.
2210 * The virtual IF bit is tweaked by CLI and STI.
2211 */
2212 IE_TO_EVENT_MASK(%edx, 4(%esp))
2213 #else
2214 sti
2215 #endif
2216 1:
2217 ret
2218 SET_SIZE(restore_int_flag)
2219 SET_SIZE(intr_restore)
2220
2221 #endif /* __i386 */
2222 #endif /* __lint */
2223
2224 #if defined(__lint)
2225
2226 void
2227 sti(void)
2228 {}
2229
2230 void
2231 cli(void)
2232 {}
2233
2234 #else /* __lint */
2235
2236 ENTRY(sti)
2237 STI
2238 ret
2239 SET_SIZE(sti)
2240
2241 ENTRY(cli)
2242 #if defined(__amd64)
2243 CLI(%rax)
2244 #elif defined(__i386)
2245 CLI(%eax)
2246 #endif /* __i386 */
2247 ret
2248 SET_SIZE(cli)
2249
2250 #endif /* __lint */
2251
2252 #if defined(__lint)
2253
2254 dtrace_icookie_t
2255 dtrace_interrupt_disable(void)
2256 { return (0); }
2257
2258 #else /* __lint */
2259
2260 #if defined(__amd64)
2261
2262 ENTRY(dtrace_interrupt_disable)
2263 pushfq
2264 popq %rax
2265 #if defined(__xpv)
2266 leaq xpv_panicking, %rdi
2267 movl (%rdi), %edi
2268 cmpl $0, %edi
2269 jne .dtrace_interrupt_disable_done
2270 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2271 /*
2272 * Synthesize the PS_IE bit from the event mask bit
2273 */
2274 andq $_BITNOT(PS_IE), %rax
2275 testb $1, %dl
2276 jnz .dtrace_interrupt_disable_done
2277 orq $PS_IE, %rax
2278 #else
2279 CLI(%rdx)
2280 #endif
2281 .dtrace_interrupt_disable_done:
2282 ret
2283 SET_SIZE(dtrace_interrupt_disable)
2284
2285 #elif defined(__i386)
2286
2287 ENTRY(dtrace_interrupt_disable)
2288 pushfl
2289 popl %eax
2290 #if defined(__xpv)
2291 leal xpv_panicking, %edx
2292 movl (%edx), %edx
2293 cmpl $0, %edx
2294 jne .dtrace_interrupt_disable_done
2295 CLIRET(%edx, %cl) /* returns event mask in %cl */
2296 /*
2297 * Synthesize the PS_IE bit from the event mask bit
2298 */
2299 andl $_BITNOT(PS_IE), %eax
2300 testb $1, %cl
2301 jnz .dtrace_interrupt_disable_done
2302 orl $PS_IE, %eax
2303 #else
2304 CLI(%edx)
2305 #endif
2306 .dtrace_interrupt_disable_done:
2307 ret
2308 SET_SIZE(dtrace_interrupt_disable)
2309
2310 #endif /* __i386 */
2311 #endif /* __lint */
2312
2313 #if defined(__lint)
2314
2315 /*ARGSUSED*/
2316 void
2317 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2318 {}
2319
2320 #else /* __lint */
2321
2322 #if defined(__amd64)
2323
2324 ENTRY(dtrace_interrupt_enable)
2325 pushq %rdi
2326 popfq
2327 #if defined(__xpv)
2328 leaq xpv_panicking, %rdx
2329 movl (%rdx), %edx
2330 cmpl $0, %edx
2331 jne .dtrace_interrupt_enable_done
2332 /*
2333 * Since we're -really- running unprivileged, our attempt
2334 * to change the state of the IF bit will be ignored. The
2335 * virtual IF bit is tweaked by CLI and STI.
2336 */
2337 IE_TO_EVENT_MASK(%rdx, %rdi)
2338 #endif
2339 .dtrace_interrupt_enable_done:
2340 ret
2341 SET_SIZE(dtrace_interrupt_enable)
2342
2343 #elif defined(__i386)
2344
2345 ENTRY(dtrace_interrupt_enable)
2346 movl 4(%esp), %eax
2347 pushl %eax
2348 popfl
2349 #if defined(__xpv)
2350 leal xpv_panicking, %edx
2351 movl (%edx), %edx
2352 cmpl $0, %edx
2353 jne .dtrace_interrupt_enable_done
2354 /*
2355 * Since we're -really- running unprivileged, our attempt
2356 * to change the state of the IF bit will be ignored. The
2357 * virtual IF bit is tweaked by CLI and STI.
2358 */
2359 IE_TO_EVENT_MASK(%edx, %eax)
2360 #endif
2361 .dtrace_interrupt_enable_done:
2362 ret
2363 SET_SIZE(dtrace_interrupt_enable)
2364
2365 #endif /* __i386 */
2366 #endif /* __lint */
2367
2368
2369 #if defined(lint)
2370
2371 void
2372 dtrace_membar_producer(void)
2373 {}
2374
2375 void
2376 dtrace_membar_consumer(void)
2377 {}
2378
2379 #else /* __lint */
2380
2381 ENTRY(dtrace_membar_producer)
2382 rep; ret /* use 2 byte return instruction when branch target */
2383 /* AMD Software Optimization Guide - Section 6.2 */
2384 SET_SIZE(dtrace_membar_producer)
2385
2386 ENTRY(dtrace_membar_consumer)
2387 rep; ret /* use 2 byte return instruction when branch target */
2388 /* AMD Software Optimization Guide - Section 6.2 */
2389 SET_SIZE(dtrace_membar_consumer)
2390
2391 #endif /* __lint */
2392
2393 #if defined(__lint)
2394
2395 kthread_id_t
2396 threadp(void)
2397 { return ((kthread_id_t)0); }
2398
2399 #else /* __lint */
2400
2401 #if defined(__amd64)
2402
2403 ENTRY(threadp)
2404 movq %gs:CPU_THREAD, %rax
2405 ret
2406 SET_SIZE(threadp)
2407
2408 #elif defined(__i386)
2409
2410 ENTRY(threadp)
2411 movl %gs:CPU_THREAD, %eax
2412 ret
2413 SET_SIZE(threadp)
2414
2415 #endif /* __i386 */
2416 #endif /* __lint */
2417
2418 /*
2419 * Checksum routine for Internet Protocol Headers
2420 */
2421
2422 #if defined(__lint)
2423
2424 /* ARGSUSED */
2425 unsigned int
2426 ip_ocsum(
2427 ushort_t *address, /* ptr to 1st message buffer */
2428 int halfword_count, /* length of data */
2429 unsigned int sum) /* partial checksum */
2430 {
2431 int i;
2432 unsigned int psum = 0; /* partial sum */
2433
2434 for (i = 0; i < halfword_count; i++, address++) {
2435 psum += *address;
2436 }
2437
2438 while ((psum >> 16) != 0) {
2439 psum = (psum & 0xffff) + (psum >> 16);
2440 }
2441
2442 psum += sum;
2443
2444 while ((psum >> 16) != 0) {
2445 psum = (psum & 0xffff) + (psum >> 16);
2446 }
2447
2448 return (psum);
2449 }
2450
2451 #else /* __lint */
2452
2453 #if defined(__amd64)
2454
2455 ENTRY(ip_ocsum)
2456 pushq %rbp
2457 movq %rsp, %rbp
2458 #ifdef DEBUG
2459 movq postbootkernelbase(%rip), %rax
2460 cmpq %rax, %rdi
2461 jnb 1f
2462 xorl %eax, %eax
2463 movq %rdi, %rsi
2464 leaq .ip_ocsum_panic_msg(%rip), %rdi
2465 call panic
2466 /*NOTREACHED*/
2467 .ip_ocsum_panic_msg:
2468 .string "ip_ocsum: address 0x%p below kernelbase\n"
2469 1:
2470 #endif
2471 movl %esi, %ecx /* halfword_count */
2472 movq %rdi, %rsi /* address */
2473 /* partial sum in %edx */
2474 xorl %eax, %eax
2475 testl %ecx, %ecx
2476 jz .ip_ocsum_done
2477 testq $3, %rsi
2478 jnz .ip_csum_notaligned
2479 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2480 .next_iter:
2481 /* XX64 opportunities for prefetch? */
2482 /* XX64 compute csum with 64 bit quantities? */
2483 subl $32, %ecx
2484 jl .less_than_32
2485
2486 addl 0(%rsi), %edx
2487 .only60:
2488 adcl 4(%rsi), %eax
2489 .only56:
2490 adcl 8(%rsi), %edx
2491 .only52:
2492 adcl 12(%rsi), %eax
2493 .only48:
2494 adcl 16(%rsi), %edx
2495 .only44:
2496 adcl 20(%rsi), %eax
2497 .only40:
2498 adcl 24(%rsi), %edx
2499 .only36:
2500 adcl 28(%rsi), %eax
2501 .only32:
2502 adcl 32(%rsi), %edx
2503 .only28:
2504 adcl 36(%rsi), %eax
2505 .only24:
2506 adcl 40(%rsi), %edx
2507 .only20:
2508 adcl 44(%rsi), %eax
2509 .only16:
2510 adcl 48(%rsi), %edx
2511 .only12:
2512 adcl 52(%rsi), %eax
2513 .only8:
2514 adcl 56(%rsi), %edx
2515 .only4:
2516 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2517 .only0:
2518 adcl $0, %eax /* could be adding -1 in eax with a carry */
2519 adcl $0, %eax
2520
2521 addq $64, %rsi
2522 testl %ecx, %ecx
2523 jnz .next_iter
2524
2525 .ip_ocsum_done:
2526 addl %eax, %edx
2527 adcl $0, %edx
2528 movl %edx, %eax /* form a 16 bit checksum by */
2529 shrl $16, %eax /* adding two halves of 32 bit checksum */
2530 addw %dx, %ax
2531 adcw $0, %ax
2532 andl $0xffff, %eax
2533 leave
2534 ret
2535
2536 .ip_csum_notaligned:
2537 xorl %edi, %edi
2538 movw (%rsi), %di
2539 addl %edi, %edx
2540 adcl $0, %edx
2541 addq $2, %rsi
2542 decl %ecx
2543 jmp .ip_csum_aligned
2544
2545 .less_than_32:
2546 addl $32, %ecx
2547 testl $1, %ecx
2548 jz .size_aligned
2549 andl $0xfe, %ecx
2550 movzwl (%rsi, %rcx, 2), %edi
2551 addl %edi, %edx
2552 adcl $0, %edx
2553 .size_aligned:
2554 movl %ecx, %edi
2555 shrl $1, %ecx
2556 shl $1, %edi
2557 subq $64, %rdi
2558 addq %rdi, %rsi
2559 leaq .ip_ocsum_jmptbl(%rip), %rdi
2560 leaq (%rdi, %rcx, 8), %rdi
2561 xorl %ecx, %ecx
2562 clc
2563 jmp *(%rdi)
2564
2565 .align 8
2566 .ip_ocsum_jmptbl:
2567 .quad .only0, .only4, .only8, .only12, .only16, .only20
2568 .quad .only24, .only28, .only32, .only36, .only40, .only44
2569 .quad .only48, .only52, .only56, .only60
2570 SET_SIZE(ip_ocsum)
2571
2572 #elif defined(__i386)
2573
2574 ENTRY(ip_ocsum)
2575 pushl %ebp
2576 movl %esp, %ebp
2577 pushl %ebx
2578 pushl %esi
2579 pushl %edi
2580 movl 12(%ebp), %ecx /* count of half words */
2581 movl 16(%ebp), %edx /* partial checksum */
2582 movl 8(%ebp), %esi
2583 xorl %eax, %eax
2584 testl %ecx, %ecx
2585 jz .ip_ocsum_done
2586
2587 testl $3, %esi
2588 jnz .ip_csum_notaligned
2589 .ip_csum_aligned:
2590 .next_iter:
2591 subl $32, %ecx
2592 jl .less_than_32
2593
2594 addl 0(%esi), %edx
2595 .only60:
2596 adcl 4(%esi), %eax
2597 .only56:
2598 adcl 8(%esi), %edx
2599 .only52:
2600 adcl 12(%esi), %eax
2601 .only48:
2602 adcl 16(%esi), %edx
2603 .only44:
2604 adcl 20(%esi), %eax
2605 .only40:
2606 adcl 24(%esi), %edx
2607 .only36:
2608 adcl 28(%esi), %eax
2609 .only32:
2610 adcl 32(%esi), %edx
2611 .only28:
2612 adcl 36(%esi), %eax
2613 .only24:
2614 adcl 40(%esi), %edx
2615 .only20:
2616 adcl 44(%esi), %eax
2617 .only16:
2618 adcl 48(%esi), %edx
2619 .only12:
2620 adcl 52(%esi), %eax
2621 .only8:
2622 adcl 56(%esi), %edx
2623 .only4:
2624 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2625 .only0:
2626 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2627 adcl $0, %eax
2628
2629 addl $64, %esi
2630 andl %ecx, %ecx
2631 jnz .next_iter
2632
2633 .ip_ocsum_done:
2634 addl %eax, %edx
2635 adcl $0, %edx
2636 movl %edx, %eax /* form a 16 bit checksum by */
2637 shrl $16, %eax /* adding two halves of 32 bit checksum */
2638 addw %dx, %ax
2639 adcw $0, %ax
2640 andl $0xffff, %eax
2641 popl %edi /* restore registers */
2642 popl %esi
2643 popl %ebx
2644 leave
2645 ret
2646
2647 .ip_csum_notaligned:
2648 xorl %edi, %edi
2649 movw (%esi), %di
2650 addl %edi, %edx
2651 adcl $0, %edx
2652 addl $2, %esi
2653 decl %ecx
2654 jmp .ip_csum_aligned
2655
2656 .less_than_32:
2657 addl $32, %ecx
2658 testl $1, %ecx
2659 jz .size_aligned
2660 andl $0xfe, %ecx
2661 movzwl (%esi, %ecx, 2), %edi
2662 addl %edi, %edx
2663 adcl $0, %edx
2664 .size_aligned:
2665 movl %ecx, %edi
2666 shrl $1, %ecx
2667 shl $1, %edi
2668 subl $64, %edi
2669 addl %edi, %esi
2670 movl $.ip_ocsum_jmptbl, %edi
2671 lea (%edi, %ecx, 4), %edi
2672 xorl %ecx, %ecx
2673 clc
2674 jmp *(%edi)
2675 SET_SIZE(ip_ocsum)
2676
2677 .data
2678 .align 4
2679
2680 .ip_ocsum_jmptbl:
2681 .long .only0, .only4, .only8, .only12, .only16, .only20
2682 .long .only24, .only28, .only32, .only36, .only40, .only44
2683 .long .only48, .only52, .only56, .only60
2684
2685
2686 #endif /* __i386 */
2687 #endif /* __lint */
2688
2689 /*
2690 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2691 * Provided to manipulate hrtime_t values.
2692 */
2693 #if defined(__lint)
2694
2695 /* result = a * b; */
2696
2697 /* ARGSUSED */
2698 unsigned long long
2699 mul32(uint_t a, uint_t b)
2700 { return (0); }
2701
2702 #else /* __lint */
2703
2704 #if defined(__amd64)
2705
2706 ENTRY(mul32)
2707 xorl %edx, %edx /* XX64 joe, paranoia? */
2708 movl %edi, %eax
2709 mull %esi
2710 shlq $32, %rdx
2711 orq %rdx, %rax
2712 ret
2713 SET_SIZE(mul32)
2714
2715 #elif defined(__i386)
2716
2717 ENTRY(mul32)
2718 movl 8(%esp), %eax
2719 movl 4(%esp), %ecx
2720 mull %ecx
2721 ret
2722 SET_SIZE(mul32)
2723
2724 #endif /* __i386 */
2725 #endif /* __lint */
2726
2727 #if defined(notused)
2728 #if defined(__lint)
2729 /* ARGSUSED */
2730 void
2731 load_pte64(uint64_t *pte, uint64_t pte_value)
2732 {}
2733 #else /* __lint */
2734 .globl load_pte64
2735 load_pte64:
2736 movl 4(%esp), %eax
2737 movl 8(%esp), %ecx
2738 movl 12(%esp), %edx
2739 movl %edx, 4(%eax)
2740 movl %ecx, (%eax)
2741 ret
2742 #endif /* __lint */
2743 #endif /* notused */
2744
2745 #if defined(__lint)
2746
2747 /*ARGSUSED*/
2748 void
2749 scan_memory(caddr_t addr, size_t size)
2750 {}
2751
2752 #else /* __lint */
2753
2754 #if defined(__amd64)
2755
2756 ENTRY(scan_memory)
2757 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2758 jz .scanm_done
2759 movq %rsi, %rcx /* move count into rep control register */
2760 movq %rdi, %rsi /* move addr into lodsq control reg. */
2761 rep lodsq /* scan the memory range */
2762 .scanm_done:
2763 rep; ret /* use 2 byte return instruction when branch target */
2764 /* AMD Software Optimization Guide - Section 6.2 */
2765 SET_SIZE(scan_memory)
2766
2767 #elif defined(__i386)
2768
2769 ENTRY(scan_memory)
2770 pushl %ecx
2771 pushl %esi
2772 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2773 shrl $2, %ecx /* convert from byte count to word count */
2774 jz .scanm_done
2775 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2776 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2777 lodsl
2778 .scanm_done:
2779 popl %esi
2780 popl %ecx
2781 ret
2782 SET_SIZE(scan_memory)
2783
2784 #endif /* __i386 */
2785 #endif /* __lint */
2786
2787
2788 #if defined(__lint)
2789
2790 /*ARGSUSED */
2791 int
2792 lowbit(ulong_t i)
2793 { return (0); }
2794
2795 #else /* __lint */
2796
2797 #if defined(__amd64)
2798
2799 ENTRY(lowbit)
2800 movl $-1, %eax
2801 bsfq %rdi, %rdi
2802 cmovnz %edi, %eax
2803 incl %eax
2804 ret
2805 SET_SIZE(lowbit)
2806
2807 #elif defined(__i386)
2808
2809 ENTRY(lowbit)
2810 bsfl 4(%esp), %eax
2811 jz 0f
2812 incl %eax
2813 ret
2814 0:
2815 xorl %eax, %eax
2816 ret
2817 SET_SIZE(lowbit)
2818
2819 #endif /* __i386 */
2820 #endif /* __lint */
2821
2822 #if defined(__lint)
2823
2824 /*ARGSUSED*/
2825 int
2826 highbit(ulong_t i)
2827 { return (0); }
2828
2829 /*ARGSUSED*/
2830 int
2831 highbit64(uint64_t i)
2832 { return (0); }
2833
2834 #else /* __lint */
2835
2836 #if defined(__amd64)
2837
2838 ENTRY(highbit)
2839 ALTENTRY(highbit64)
2840 movl $-1, %eax
2841 bsrq %rdi, %rdi
2842 cmovnz %edi, %eax
2843 incl %eax
2844 ret
2845 SET_SIZE(highbit64)
2846 SET_SIZE(highbit)
2847
2848 #elif defined(__i386)
2849
2850 ENTRY(highbit)
2851 bsrl 4(%esp), %eax
2852 jz 0f
2853 incl %eax
2854 ret
2855 0:
2856 xorl %eax, %eax
2857 ret
2858 SET_SIZE(highbit)
2859
2860 ENTRY(highbit64)
2861 bsrl 8(%esp), %eax
2862 jz highbit
2863 addl $33, %eax
2864 ret
2865 SET_SIZE(highbit64)
2866
2867 #endif /* __i386 */
2868 #endif /* __lint */
2869
2870 #if defined(__lint)
2871
2872 /*ARGSUSED*/
2873 uint64_t
2874 rdmsr(uint_t r)
2875 { return (0); }
2876
2877 /*ARGSUSED*/
2878 void
2879 wrmsr(uint_t r, const uint64_t val)
2880 {}
2881
2882 /*ARGSUSED*/
2883 uint64_t
2884 xrdmsr(uint_t r)
2885 { return (0); }
2886
2887 /*ARGSUSED*/
2888 void
2889 xwrmsr(uint_t r, const uint64_t val)
2890 {}
2891
2892 void
2893 invalidate_cache(void)
2894 {}
2895
2896 /*ARGSUSED*/
2897 uint64_t
2898 get_xcr(uint_t r)
2899 { return (0); }
2900
2901 /*ARGSUSED*/
2902 void
2903 set_xcr(uint_t r, const uint64_t val)
2904 {}
2905
2906 #else /* __lint */
2907
2908 #define XMSR_ACCESS_VAL $0x9c5a203a
2909
2910 #if defined(__amd64)
2911
2912 ENTRY(rdmsr)
2913 movl %edi, %ecx
2914 rdmsr
2915 shlq $32, %rdx
2916 orq %rdx, %rax
2917 ret
2918 SET_SIZE(rdmsr)
2919
2920 ENTRY(wrmsr)
2921 movq %rsi, %rdx
2922 shrq $32, %rdx
2923 movl %esi, %eax
2924 movl %edi, %ecx
2925 wrmsr
2926 ret
2927 SET_SIZE(wrmsr)
2928
2929 ENTRY(xrdmsr)
2930 pushq %rbp
2931 movq %rsp, %rbp
2932 movl %edi, %ecx
2933 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2934 rdmsr
2935 shlq $32, %rdx
2936 orq %rdx, %rax
2937 leave
2938 ret
2939 SET_SIZE(xrdmsr)
2940
2941 ENTRY(xwrmsr)
2942 pushq %rbp
2943 movq %rsp, %rbp
2944 movl %edi, %ecx
2945 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2946 movq %rsi, %rdx
2947 shrq $32, %rdx
2948 movl %esi, %eax
2949 wrmsr
2950 leave
2951 ret
2952 SET_SIZE(xwrmsr)
2953
2954 ENTRY(get_xcr)
2955 movl %edi, %ecx
2956 #xgetbv
2957 .byte 0x0f,0x01,0xd0
2958 shlq $32, %rdx
2959 orq %rdx, %rax
2960 ret
2961 SET_SIZE(get_xcr)
2962
2963 ENTRY(set_xcr)
2964 movq %rsi, %rdx
2965 shrq $32, %rdx
2966 movl %esi, %eax
2967 movl %edi, %ecx
2968 #xsetbv
2969 .byte 0x0f,0x01,0xd1
2970 ret
2971 SET_SIZE(set_xcr)
2972
2973 #elif defined(__i386)
2974
2975 ENTRY(rdmsr)
2976 movl 4(%esp), %ecx
2977 rdmsr
2978 ret
2979 SET_SIZE(rdmsr)
2980
2981 ENTRY(wrmsr)
2982 movl 4(%esp), %ecx
2983 movl 8(%esp), %eax
2984 movl 12(%esp), %edx
2985 wrmsr
2986 ret
2987 SET_SIZE(wrmsr)
2988
2989 ENTRY(xrdmsr)
2990 pushl %ebp
2991 movl %esp, %ebp
2992 movl 8(%esp), %ecx
2993 pushl %edi
2994 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2995 rdmsr
2996 popl %edi
2997 leave
2998 ret
2999 SET_SIZE(xrdmsr)
3000
3001 ENTRY(xwrmsr)
3002 pushl %ebp
3003 movl %esp, %ebp
3004 movl 8(%esp), %ecx
3005 movl 12(%esp), %eax
3006 movl 16(%esp), %edx
3007 pushl %edi
3008 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3009 wrmsr
3010 popl %edi
3011 leave
3012 ret
3013 SET_SIZE(xwrmsr)
3014
3015 ENTRY(get_xcr)
3016 movl 4(%esp), %ecx
3017 #xgetbv
3018 .byte 0x0f,0x01,0xd0
3019 ret
3020 SET_SIZE(get_xcr)
3021
3022 ENTRY(set_xcr)
3023 movl 4(%esp), %ecx
3024 movl 8(%esp), %eax
3025 movl 12(%esp), %edx
3026 #xsetbv
3027 .byte 0x0f,0x01,0xd1
3028 ret
3029 SET_SIZE(set_xcr)
3030
3031 #endif /* __i386 */
3032
3033 ENTRY(invalidate_cache)
3034 wbinvd
3035 ret
3036 SET_SIZE(invalidate_cache)
3037
3038 #endif /* __lint */
3039
3040 #if defined(__lint)
3041
3042 /*ARGSUSED*/
3043 void
3044 getcregs(struct cregs *crp)
3045 {}
3046
3047 #else /* __lint */
3048
3049 #if defined(__amd64)
3050
3051 ENTRY_NP(getcregs)
3052 #if defined(__xpv)
3053 /*
3054 * Only a few of the hardware control registers or descriptor tables
3055 * are directly accessible to us, so just zero the structure.
3056 *
3057 * XXPV Perhaps it would be helpful for the hypervisor to return
3058 * virtualized versions of these for post-mortem use.
3059 * (Need to reevaluate - perhaps it already does!)
3060 */
3061 pushq %rdi /* save *crp */
3062 movq $CREGSZ, %rsi
3063 call bzero
3064 popq %rdi
3065
3066 /*
3067 * Dump what limited information we can
3068 */
3069 movq %cr0, %rax
3070 movq %rax, CREG_CR0(%rdi) /* cr0 */
3071 movq %cr2, %rax
3072 movq %rax, CREG_CR2(%rdi) /* cr2 */
3073 movq %cr3, %rax
3074 movq %rax, CREG_CR3(%rdi) /* cr3 */
3075 movq %cr4, %rax
3076 movq %rax, CREG_CR4(%rdi) /* cr4 */
3077
3078 #else /* __xpv */
3079
3080 #define GETMSR(r, off, d) \
3081 movl $r, %ecx; \
3082 rdmsr; \
3083 movl %eax, off(d); \
3084 movl %edx, off+4(d)
3085
3086 xorl %eax, %eax
3087 movq %rax, CREG_GDT+8(%rdi)
3088 sgdt CREG_GDT(%rdi) /* 10 bytes */
3089 movq %rax, CREG_IDT+8(%rdi)
3090 sidt CREG_IDT(%rdi) /* 10 bytes */
3091 movq %rax, CREG_LDT(%rdi)
3092 sldt CREG_LDT(%rdi) /* 2 bytes */
3093 movq %rax, CREG_TASKR(%rdi)
3094 str CREG_TASKR(%rdi) /* 2 bytes */
3095 movq %cr0, %rax
3096 movq %rax, CREG_CR0(%rdi) /* cr0 */
3097 movq %cr2, %rax
3098 movq %rax, CREG_CR2(%rdi) /* cr2 */
3099 movq %cr3, %rax
3100 movq %rax, CREG_CR3(%rdi) /* cr3 */
3101 movq %cr4, %rax
3102 movq %rax, CREG_CR4(%rdi) /* cr4 */
3103 movq %cr8, %rax
3104 movq %rax, CREG_CR8(%rdi) /* cr8 */
3105 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3106 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3107 #endif /* __xpv */
3108 ret
3109 SET_SIZE(getcregs)
3110
3111 #undef GETMSR
3112
3113 #elif defined(__i386)
3114
3115 ENTRY_NP(getcregs)
3116 #if defined(__xpv)
3117 /*
3118 * Only a few of the hardware control registers or descriptor tables
3119 * are directly accessible to us, so just zero the structure.
3120 *
3121 * XXPV Perhaps it would be helpful for the hypervisor to return
3122 * virtualized versions of these for post-mortem use.
3123 * (Need to reevaluate - perhaps it already does!)
3124 */
3125 movl 4(%esp), %edx
3126 pushl $CREGSZ
3127 pushl %edx
3128 call bzero
3129 addl $8, %esp
3130 movl 4(%esp), %edx
3131
3132 /*
3133 * Dump what limited information we can
3134 */
3135 movl %cr0, %eax
3136 movl %eax, CREG_CR0(%edx) /* cr0 */
3137 movl %cr2, %eax
3138 movl %eax, CREG_CR2(%edx) /* cr2 */
3139 movl %cr3, %eax
3140 movl %eax, CREG_CR3(%edx) /* cr3 */
3141 movl %cr4, %eax
3142 movl %eax, CREG_CR4(%edx) /* cr4 */
3143
3144 #else /* __xpv */
3145
3146 movl 4(%esp), %edx
3147 movw $0, CREG_GDT+6(%edx)
3148 movw $0, CREG_IDT+6(%edx)
3149 sgdt CREG_GDT(%edx) /* gdt */
3150 sidt CREG_IDT(%edx) /* idt */
3151 sldt CREG_LDT(%edx) /* ldt */
3152 str CREG_TASKR(%edx) /* task */
3153 movl %cr0, %eax
3154 movl %eax, CREG_CR0(%edx) /* cr0 */
3155 movl %cr2, %eax
3156 movl %eax, CREG_CR2(%edx) /* cr2 */
3157 movl %cr3, %eax
3158 movl %eax, CREG_CR3(%edx) /* cr3 */
3159 bt $X86FSET_LARGEPAGE, x86_featureset
3160 jnc .nocr4
3161 movl %cr4, %eax
3162 movl %eax, CREG_CR4(%edx) /* cr4 */
3163 jmp .skip
3164 .nocr4:
3165 movl $0, CREG_CR4(%edx)
3166 .skip:
3167 #endif
3168 ret
3169 SET_SIZE(getcregs)
3170
3171 #endif /* __i386 */
3172 #endif /* __lint */
3173
3174
3175 /*
3176 * A panic trigger is a word which is updated atomically and can only be set
3177 * once. We atomically store 0xDEFACEDD and load the old value. If the
3178 * previous value was 0, we succeed and return 1; otherwise return 0.
3179 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3180 * has its own version of this function to allow it to panic correctly from
3181 * probe context.
3182 */
3183 #if defined(__lint)
3184
3185 /*ARGSUSED*/
3186 int
3187 panic_trigger(int *tp)
3188 { return (0); }
3189
3190 /*ARGSUSED*/
3191 int
3192 dtrace_panic_trigger(int *tp)
3193 { return (0); }
3194
3195 #else /* __lint */
3196
3197 #if defined(__amd64)
3198
3199 ENTRY_NP(panic_trigger)
3200 xorl %eax, %eax
3201 movl $0xdefacedd, %edx
3202 lock
3203 xchgl %edx, (%rdi)
3204 cmpl $0, %edx
3205 je 0f
3206 movl $0, %eax
3207 ret
3208 0: movl $1, %eax
3209 ret
3210 SET_SIZE(panic_trigger)
3211
3212 ENTRY_NP(dtrace_panic_trigger)
3213 xorl %eax, %eax
3214 movl $0xdefacedd, %edx
3215 lock
3216 xchgl %edx, (%rdi)
3217 cmpl $0, %edx
3218 je 0f
3219 movl $0, %eax
3220 ret
3221 0: movl $1, %eax
3222 ret
3223 SET_SIZE(dtrace_panic_trigger)
3224
3225 #elif defined(__i386)
3226
3227 ENTRY_NP(panic_trigger)
3228 movl 4(%esp), %edx / %edx = address of trigger
3229 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3230 lock / assert lock
3231 xchgl %eax, (%edx) / exchange %eax and the trigger
3232 cmpl $0, %eax / if (%eax == 0x0)
3233 je 0f / return (1);
3234 movl $0, %eax / else
3235 ret / return (0);
3236 0: movl $1, %eax
3237 ret
3238 SET_SIZE(panic_trigger)
3239
3240 ENTRY_NP(dtrace_panic_trigger)
3241 movl 4(%esp), %edx / %edx = address of trigger
3242 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3243 lock / assert lock
3244 xchgl %eax, (%edx) / exchange %eax and the trigger
3245 cmpl $0, %eax / if (%eax == 0x0)
3246 je 0f / return (1);
3247 movl $0, %eax / else
3248 ret / return (0);
3249 0: movl $1, %eax
3250 ret
3251 SET_SIZE(dtrace_panic_trigger)
3252
3253 #endif /* __i386 */
3254 #endif /* __lint */
3255
3256 /*
3257 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3258 * into the panic code implemented in panicsys(). vpanic() is responsible
3259 * for passing through the format string and arguments, and constructing a
3260 * regs structure on the stack into which it saves the current register
3261 * values. If we are not dying due to a fatal trap, these registers will
3262 * then be preserved in panicbuf as the current processor state. Before
3263 * invoking panicsys(), vpanic() activates the first panic trigger (see
3264 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3265 * DTrace takes a slightly different panic path if it must panic from probe
3266 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3267 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3268 * branches back into vpanic().
3269 */
3270 #if defined(__lint)
3271
3272 /*ARGSUSED*/
3273 void
3274 vpanic(const char *format, va_list alist)
3275 {}
3276
3277 /*ARGSUSED*/
3278 void
3279 dtrace_vpanic(const char *format, va_list alist)
3280 {}
3281
3282 #else /* __lint */
3283
3284 #if defined(__amd64)
3285
3286 ENTRY_NP(vpanic) /* Initial stack layout: */
3287
3288 pushq %rbp /* | %rip | 0x60 */
3289 movq %rsp, %rbp /* | %rbp | 0x58 */
3290 pushfq /* | rfl | 0x50 */
3291 pushq %r11 /* | %r11 | 0x48 */
3292 pushq %r10 /* | %r10 | 0x40 */
3293 pushq %rbx /* | %rbx | 0x38 */
3294 pushq %rax /* | %rax | 0x30 */
3295 pushq %r9 /* | %r9 | 0x28 */
3296 pushq %r8 /* | %r8 | 0x20 */
3297 pushq %rcx /* | %rcx | 0x18 */
3298 pushq %rdx /* | %rdx | 0x10 */
3299 pushq %rsi /* | %rsi | 0x8 alist */
3300 pushq %rdi /* | %rdi | 0x0 format */
3301
3302 movq %rsp, %rbx /* %rbx = current %rsp */
3303
3304 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3305 call panic_trigger /* %eax = panic_trigger() */
3306
3307 vpanic_common:
3308 /*
3309 * The panic_trigger result is in %eax from the call above, and
3310 * dtrace_panic places it in %eax before branching here.
3311 * The rdmsr instructions that follow below will clobber %eax so
3312 * we stash the panic_trigger result in %r11d.
3313 */
3314 movl %eax, %r11d
3315 cmpl $0, %r11d
3316 je 0f
3317
3318 /*
3319 * If panic_trigger() was successful, we are the first to initiate a
3320 * panic: we now switch to the reserved panic_stack before continuing.
3321 */
3322 leaq panic_stack(%rip), %rsp
3323 addq $PANICSTKSIZE, %rsp
3324 0: subq $REGSIZE, %rsp
3325 /*
3326 * Now that we've got everything set up, store the register values as
3327 * they were when we entered vpanic() to the designated location in
3328 * the regs structure we allocated on the stack.
3329 */
3330 movq 0x0(%rbx), %rcx
3331 movq %rcx, REGOFF_RDI(%rsp)
3332 movq 0x8(%rbx), %rcx
3333 movq %rcx, REGOFF_RSI(%rsp)
3334 movq 0x10(%rbx), %rcx
3335 movq %rcx, REGOFF_RDX(%rsp)
3336 movq 0x18(%rbx), %rcx
3337 movq %rcx, REGOFF_RCX(%rsp)
3338 movq 0x20(%rbx), %rcx
3339
3340 movq %rcx, REGOFF_R8(%rsp)
3341 movq 0x28(%rbx), %rcx
3342 movq %rcx, REGOFF_R9(%rsp)
3343 movq 0x30(%rbx), %rcx
3344 movq %rcx, REGOFF_RAX(%rsp)
3345 movq 0x38(%rbx), %rcx
3346 movq %rcx, REGOFF_RBX(%rsp)
3347 movq 0x58(%rbx), %rcx
3348
3349 movq %rcx, REGOFF_RBP(%rsp)
3350 movq 0x40(%rbx), %rcx
3351 movq %rcx, REGOFF_R10(%rsp)
3352 movq 0x48(%rbx), %rcx
3353 movq %rcx, REGOFF_R11(%rsp)
3354 movq %r12, REGOFF_R12(%rsp)
3355
3356 movq %r13, REGOFF_R13(%rsp)
3357 movq %r14, REGOFF_R14(%rsp)
3358 movq %r15, REGOFF_R15(%rsp)
3359
3360 xorl %ecx, %ecx
3361 movw %ds, %cx
3362 movq %rcx, REGOFF_DS(%rsp)
3363 movw %es, %cx
3364 movq %rcx, REGOFF_ES(%rsp)
3365 movw %fs, %cx
3366 movq %rcx, REGOFF_FS(%rsp)
3367 movw %gs, %cx
3368 movq %rcx, REGOFF_GS(%rsp)
3369
3370 movq $0, REGOFF_TRAPNO(%rsp)
3371
3372 movq $0, REGOFF_ERR(%rsp)
3373 leaq vpanic(%rip), %rcx
3374 movq %rcx, REGOFF_RIP(%rsp)
3375 movw %cs, %cx
3376 movzwq %cx, %rcx
3377 movq %rcx, REGOFF_CS(%rsp)
3378 movq 0x50(%rbx), %rcx
3379 movq %rcx, REGOFF_RFL(%rsp)
3380 movq %rbx, %rcx
3381 addq $0x60, %rcx
3382 movq %rcx, REGOFF_RSP(%rsp)
3383 movw %ss, %cx
3384 movzwq %cx, %rcx
3385 movq %rcx, REGOFF_SS(%rsp)
3386
3387 /*
3388 * panicsys(format, alist, rp, on_panic_stack)
3389 */
3390 movq REGOFF_RDI(%rsp), %rdi /* format */
3391 movq REGOFF_RSI(%rsp), %rsi /* alist */
3392 movq %rsp, %rdx /* struct regs */
3393 movl %r11d, %ecx /* on_panic_stack */
3394 call panicsys
3395 addq $REGSIZE, %rsp
3396 popq %rdi
3397 popq %rsi
3398 popq %rdx
3399 popq %rcx
3400 popq %r8
3401 popq %r9
3402 popq %rax
3403 popq %rbx
3404 popq %r10
3405 popq %r11
3406 popfq
3407 leave
3408 ret
3409 SET_SIZE(vpanic)
3410
3411 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3412
3413 pushq %rbp /* | %rip | 0x60 */
3414 movq %rsp, %rbp /* | %rbp | 0x58 */
3415 pushfq /* | rfl | 0x50 */
3416 pushq %r11 /* | %r11 | 0x48 */
3417 pushq %r10 /* | %r10 | 0x40 */
3418 pushq %rbx /* | %rbx | 0x38 */
3419 pushq %rax /* | %rax | 0x30 */
3420 pushq %r9 /* | %r9 | 0x28 */
3421 pushq %r8 /* | %r8 | 0x20 */
3422 pushq %rcx /* | %rcx | 0x18 */
3423 pushq %rdx /* | %rdx | 0x10 */
3424 pushq %rsi /* | %rsi | 0x8 alist */
3425 pushq %rdi /* | %rdi | 0x0 format */
3426
3427 movq %rsp, %rbx /* %rbx = current %rsp */
3428
3429 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3430 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3431 jmp vpanic_common
3432
3433 SET_SIZE(dtrace_vpanic)
3434
3435 #elif defined(__i386)
3436
3437 ENTRY_NP(vpanic) / Initial stack layout:
3438
3439 pushl %ebp / | %eip | 20
3440 movl %esp, %ebp / | %ebp | 16
3441 pushl %eax / | %eax | 12
3442 pushl %ebx / | %ebx | 8
3443 pushl %ecx / | %ecx | 4
3444 pushl %edx / | %edx | 0
3445
3446 movl %esp, %ebx / %ebx = current stack pointer
3447
3448 lea panic_quiesce, %eax / %eax = &panic_quiesce
3449 pushl %eax / push &panic_quiesce
3450 call panic_trigger / %eax = panic_trigger()
3451 addl $4, %esp / reset stack pointer
3452
3453 vpanic_common:
3454 cmpl $0, %eax / if (%eax == 0)
3455 je 0f / goto 0f;
3456
3457 /*
3458 * If panic_trigger() was successful, we are the first to initiate a
3459 * panic: we now switch to the reserved panic_stack before continuing.
3460 */
3461 lea panic_stack, %esp / %esp = panic_stack
3462 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3463
3464 0: subl $REGSIZE, %esp / allocate struct regs
3465
3466 /*
3467 * Now that we've got everything set up, store the register values as
3468 * they were when we entered vpanic() to the designated location in
3469 * the regs structure we allocated on the stack.
3470 */
3471 #if !defined(__GNUC_AS__)
3472 movw %gs, %edx
3473 movl %edx, REGOFF_GS(%esp)
3474 movw %fs, %edx
3475 movl %edx, REGOFF_FS(%esp)
3476 movw %es, %edx
3477 movl %edx, REGOFF_ES(%esp)
3478 movw %ds, %edx
3479 movl %edx, REGOFF_DS(%esp)
3480 #else /* __GNUC_AS__ */
3481 mov %gs, %edx
3482 mov %edx, REGOFF_GS(%esp)
3483 mov %fs, %edx
3484 mov %edx, REGOFF_FS(%esp)
3485 mov %es, %edx
3486 mov %edx, REGOFF_ES(%esp)
3487 mov %ds, %edx
3488 mov %edx, REGOFF_DS(%esp)
3489 #endif /* __GNUC_AS__ */
3490 movl %edi, REGOFF_EDI(%esp)
3491 movl %esi, REGOFF_ESI(%esp)
3492 movl 16(%ebx), %ecx
3493 movl %ecx, REGOFF_EBP(%esp)
3494 movl %ebx, %ecx
3495 addl $20, %ecx
3496 movl %ecx, REGOFF_ESP(%esp)
3497 movl 8(%ebx), %ecx
3498 movl %ecx, REGOFF_EBX(%esp)
3499 movl 0(%ebx), %ecx
3500 movl %ecx, REGOFF_EDX(%esp)
3501 movl 4(%ebx), %ecx
3502 movl %ecx, REGOFF_ECX(%esp)
3503 movl 12(%ebx), %ecx
3504 movl %ecx, REGOFF_EAX(%esp)
3505 movl $0, REGOFF_TRAPNO(%esp)
3506 movl $0, REGOFF_ERR(%esp)
3507 lea vpanic, %ecx
3508 movl %ecx, REGOFF_EIP(%esp)
3509 #if !defined(__GNUC_AS__)
3510 movw %cs, %edx
3511 #else /* __GNUC_AS__ */
3512 mov %cs, %edx
3513 #endif /* __GNUC_AS__ */
3514 movl %edx, REGOFF_CS(%esp)
3515 pushfl
3516 popl %ecx
3517 #if defined(__xpv)
3518 /*
3519 * Synthesize the PS_IE bit from the event mask bit
3520 */
3521 CURTHREAD(%edx)
3522 KPREEMPT_DISABLE(%edx)
3523 EVENT_MASK_TO_IE(%edx, %ecx)
3524 CURTHREAD(%edx)
3525 KPREEMPT_ENABLE_NOKP(%edx)
3526 #endif
3527 movl %ecx, REGOFF_EFL(%esp)
3528 movl $0, REGOFF_UESP(%esp)
3529 #if !defined(__GNUC_AS__)
3530 movw %ss, %edx
3531 #else /* __GNUC_AS__ */
3532 mov %ss, %edx
3533 #endif /* __GNUC_AS__ */
3534 movl %edx, REGOFF_SS(%esp)
3535
3536 movl %esp, %ecx / %ecx = ®s
3537 pushl %eax / push on_panic_stack
3538 pushl %ecx / push ®s
3539 movl 12(%ebp), %ecx / %ecx = alist
3540 pushl %ecx / push alist
3541 movl 8(%ebp), %ecx / %ecx = format
3542 pushl %ecx / push format
3543 call panicsys / panicsys();
3544 addl $16, %esp / pop arguments
3545
3546 addl $REGSIZE, %esp
3547 popl %edx
3548 popl %ecx
3549 popl %ebx
3550 popl %eax
3551 leave
3552 ret
3553 SET_SIZE(vpanic)
3554
3555 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3556
3557 pushl %ebp / | %eip | 20
3558 movl %esp, %ebp / | %ebp | 16
3559 pushl %eax / | %eax | 12
3560 pushl %ebx / | %ebx | 8
3561 pushl %ecx / | %ecx | 4
3562 pushl %edx / | %edx | 0
3563
3564 movl %esp, %ebx / %ebx = current stack pointer
3565
3566 lea panic_quiesce, %eax / %eax = &panic_quiesce
3567 pushl %eax / push &panic_quiesce
3568 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3569 addl $4, %esp / reset stack pointer
3570 jmp vpanic_common / jump back to common code
3571
3572 SET_SIZE(dtrace_vpanic)
3573
3574 #endif /* __i386 */
3575 #endif /* __lint */
3576
3577 #if defined(__lint)
3578
3579 void
3580 hres_tick(void)
3581 {}
3582
3583 int64_t timedelta;
3584 hrtime_t hrtime_base;
3585
3586 #else /* __lint */
3587
3588 DGDEF3(timedelta, 8, 8)
3589 .long 0, 0
3590
3591 /*
3592 * initialized to a non zero value to make pc_gethrtime()
3593 * work correctly even before clock is initialized
3594 */
3595 DGDEF3(hrtime_base, 8, 8)
3596 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3597
3598 DGDEF3(adj_shift, 4, 4)
3599 .long ADJ_SHIFT
3600
3601 #if defined(__amd64)
3602
3603 ENTRY_NP(hres_tick)
3604 pushq %rbp
3605 movq %rsp, %rbp
3606
3607 /*
3608 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3609 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3610 * At worst, performing this now instead of under CLOCK_LOCK may
3611 * introduce some jitter in pc_gethrestime().
3612 */
3613 call *gethrtimef(%rip)
3614 movq %rax, %r8
3615
3616 leaq hres_lock(%rip), %rax
3617 movb $-1, %dl
3618 .CL1:
3619 xchgb %dl, (%rax)
3620 testb %dl, %dl
3621 jz .CL3 /* got it */
3622 .CL2:
3623 cmpb $0, (%rax) /* possible to get lock? */
3624 pause
3625 jne .CL2
3626 jmp .CL1 /* yes, try again */
3627 .CL3:
3628 /*
3629 * compute the interval since last time hres_tick was called
3630 * and adjust hrtime_base and hrestime accordingly
3631 * hrtime_base is an 8 byte value (in nsec), hrestime is
3632 * a timestruc_t (sec, nsec)
3633 */
3634 leaq hres_last_tick(%rip), %rax
3635 movq %r8, %r11
3636 subq (%rax), %r8
3637 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3638 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3639 /*
3640 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3641 */
3642 movq %r11, (%rax)
3643
3644 call __adj_hrestime
3645
3646 /*
3647 * release the hres_lock
3648 */
3649 incl hres_lock(%rip)
3650 leave
3651 ret
3652 SET_SIZE(hres_tick)
3653
3654 #elif defined(__i386)
3655
3656 ENTRY_NP(hres_tick)
3657 pushl %ebp
3658 movl %esp, %ebp
3659 pushl %esi
3660 pushl %ebx
3661
3662 /*
3663 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3664 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3665 * At worst, performing this now instead of under CLOCK_LOCK may
3666 * introduce some jitter in pc_gethrestime().
3667 */
3668 call *gethrtimef
3669 movl %eax, %ebx
3670 movl %edx, %esi
3671
3672 movl $hres_lock, %eax
3673 movl $-1, %edx
3674 .CL1:
3675 xchgb %dl, (%eax)
3676 testb %dl, %dl
3677 jz .CL3 / got it
3678 .CL2:
3679 cmpb $0, (%eax) / possible to get lock?
3680 pause
3681 jne .CL2
3682 jmp .CL1 / yes, try again
3683 .CL3:
3684 /*
3685 * compute the interval since last time hres_tick was called
3686 * and adjust hrtime_base and hrestime accordingly
3687 * hrtime_base is an 8 byte value (in nsec), hrestime is
3688 * timestruc_t (sec, nsec)
3689 */
3690
3691 lea hres_last_tick, %eax
3692
3693 movl %ebx, %edx
3694 movl %esi, %ecx
3695
3696 subl (%eax), %edx
3697 sbbl 4(%eax), %ecx
3698
3699 addl %edx, hrtime_base / add interval to hrtime_base
3700 adcl %ecx, hrtime_base+4
3701
3702 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3703
3704 /
3705 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3706 /
3707 movl %ebx, (%eax)
3708 movl %esi, 4(%eax)
3709
3710 / get hrestime at this moment. used as base for pc_gethrestime
3711 /
3712 / Apply adjustment, if any
3713 /
3714 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3715 / (max_hres_adj)
3716 /
3717 / void
3718 / adj_hrestime()
3719 / {
3720 / long long adj;
3721 /
3722 / if (hrestime_adj == 0)
3723 / adj = 0;
3724 / else if (hrestime_adj > 0) {
3725 / if (hrestime_adj < HRES_ADJ)
3726 / adj = hrestime_adj;
3727 / else
3728 / adj = HRES_ADJ;
3729 / }
3730 / else {
3731 / if (hrestime_adj < -(HRES_ADJ))
3732 / adj = -(HRES_ADJ);
3733 / else
3734 / adj = hrestime_adj;
3735 / }
3736 /
3737 / timedelta -= adj;
3738 / hrestime_adj = timedelta;
3739 / hrestime.tv_nsec += adj;
3740 /
3741 / while (hrestime.tv_nsec >= NANOSEC) {
3742 / one_sec++;
3743 / hrestime.tv_sec++;
3744 / hrestime.tv_nsec -= NANOSEC;
3745 / }
3746 / }
3747 __adj_hrestime:
3748 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3749 movl hrestime_adj+4, %edx
3750 andl %esi, %esi
3751 jne .CL4 / no
3752 andl %edx, %edx
3753 jne .CL4 / no
3754 subl %ecx, %ecx / yes, adj = 0;
3755 subl %edx, %edx
3756 jmp .CL5
3757 .CL4:
3758 subl %ecx, %ecx
3759 subl %eax, %eax
3760 subl %esi, %ecx
3761 sbbl %edx, %eax
3762 andl %eax, %eax / if (hrestime_adj > 0)
3763 jge .CL6
3764
3765 / In the following comments, HRES_ADJ is used, while in the code
3766 / max_hres_adj is used.
3767 /
3768 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3769 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3770 / on the logical equivalence of:
3771 /
3772 / !(hrestime_adj < HRES_ADJ)
3773 /
3774 / and the two step sequence:
3775 /
3776 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3777 /
3778 / which computes whether or not the least significant 32-bits
3779 / of hrestime_adj is greater than HRES_ADJ, followed by:
3780 /
3781 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3782 /
3783 / which generates a carry whenever step 1 is true or the most
3784 / significant long of the longlong hrestime_adj is non-zero.
3785
3786 movl max_hres_adj, %ecx / hrestime_adj is positive
3787 subl %esi, %ecx
3788 movl %edx, %eax
3789 adcl $-1, %eax
3790 jnc .CL7
3791 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3792 subl %edx, %edx
3793 jmp .CL5
3794
3795 / The following computation is similar to the one above.
3796 /
3797 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3798 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3799 / on the logical equivalence of:
3800 /
3801 / (hrestime_adj > -HRES_ADJ)
3802 /
3803 / and the two step sequence:
3804 /
3805 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3806 /
3807 / which means the least significant 32-bits of hrestime_adj is
3808 / greater than -HRES_ADJ, followed by:
3809 /
3810 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3811 /
3812 / which generates a carry only when step 1 is true and the most
3813 / significant long of the longlong hrestime_adj is -1.
3814
3815 .CL6: / hrestime_adj is negative
3816 movl %esi, %ecx
3817 addl max_hres_adj, %ecx
3818 movl %edx, %eax
3819 adcl $0, %eax
3820 jc .CL7
3821 xor %ecx, %ecx
3822 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3823 movl $-1, %edx
3824 jmp .CL5
3825 .CL7:
3826 movl %esi, %ecx / adj = hrestime_adj;
3827 .CL5:
3828 movl timedelta, %esi
3829 subl %ecx, %esi
3830 movl timedelta+4, %eax
3831 sbbl %edx, %eax
3832 movl %esi, timedelta
3833 movl %eax, timedelta+4 / timedelta -= adj;
3834 movl %esi, hrestime_adj
3835 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3836 addl hrestime+4, %ecx
3837
3838 movl %ecx, %eax / eax = tv_nsec
3839 1:
3840 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3841 jb .CL8 / no
3842 incl one_sec / yes, one_sec++;
3843 incl hrestime / hrestime.tv_sec++;
3844 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3845 jmp 1b / check for more seconds
3846
3847 .CL8:
3848 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3849 incl hres_lock / release the hres_lock
3850
3851 popl %ebx
3852 popl %esi
3853 leave
3854 ret
3855 SET_SIZE(hres_tick)
3856
3857 #endif /* __i386 */
3858 #endif /* __lint */
3859
3860 /*
3861 * void prefetch_smap_w(void *)
3862 *
3863 * Prefetch ahead within a linear list of smap structures.
3864 * Not implemented for ia32. Stub for compatibility.
3865 */
3866
3867 #if defined(__lint)
3868
3869 /*ARGSUSED*/
3870 void prefetch_smap_w(void *smp)
3871 {}
3872
3873 #else /* __lint */
3874
3875 ENTRY(prefetch_smap_w)
3876 rep; ret /* use 2 byte return instruction when branch target */
3877 /* AMD Software Optimization Guide - Section 6.2 */
3878 SET_SIZE(prefetch_smap_w)
3879
3880 #endif /* __lint */
3881
3882 /*
3883 * prefetch_page_r(page_t *)
3884 * issue prefetch instructions for a page_t
3885 */
3886 #if defined(__lint)
3887
3888 /*ARGSUSED*/
3889 void
3890 prefetch_page_r(void *pp)
3891 {}
3892
3893 #else /* __lint */
3894
3895 ENTRY(prefetch_page_r)
3896 rep; ret /* use 2 byte return instruction when branch target */
3897 /* AMD Software Optimization Guide - Section 6.2 */
3898 SET_SIZE(prefetch_page_r)
3899
3900 #endif /* __lint */
3901
3902 #if defined(__lint)
3903
3904 /*ARGSUSED*/
3905 int
3906 bcmp(const void *s1, const void *s2, size_t count)
3907 { return (0); }
3908
3909 #else /* __lint */
3910
3911 #if defined(__amd64)
3912
3913 ENTRY(bcmp)
3914 pushq %rbp
3915 movq %rsp, %rbp
3916 #ifdef DEBUG
3917 testq %rdx,%rdx
3918 je 1f
3919 movq postbootkernelbase(%rip), %r11
3920 cmpq %r11, %rdi
3921 jb 0f
3922 cmpq %r11, %rsi
3923 jnb 1f
3924 0: leaq .bcmp_panic_msg(%rip), %rdi
3925 xorl %eax, %eax
3926 call panic
3927 1:
3928 #endif /* DEBUG */
3929 call memcmp
3930 testl %eax, %eax
3931 setne %dl
3932 leave
3933 movzbl %dl, %eax
3934 ret
3935 SET_SIZE(bcmp)
3936
3937 #elif defined(__i386)
3938
3939 #define ARG_S1 8
3940 #define ARG_S2 12
3941 #define ARG_LENGTH 16
3942
3943 ENTRY(bcmp)
3944 pushl %ebp
3945 movl %esp, %ebp / create new stack frame
3946 #ifdef DEBUG
3947 cmpl $0, ARG_LENGTH(%ebp)
3948 je 1f
3949 movl postbootkernelbase, %eax
3950 cmpl %eax, ARG_S1(%ebp)
3951 jb 0f
3952 cmpl %eax, ARG_S2(%ebp)
3953 jnb 1f
3954 0: pushl $.bcmp_panic_msg
3955 call panic
3956 1:
3957 #endif /* DEBUG */
3958
3959 pushl %edi / save register variable
3960 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3961 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3962 cmpl %eax, %ecx / if the same string
3963 je .equal / goto .equal
3964 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3965 cmpl $4, %edi / if %edi < 4
3966 jb .byte_check / goto .byte_check
3967 .align 4
3968 .word_loop:
3969 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3970 leal -4(%edi), %edi / %edi -= 4
3971 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3972 jne .word_not_equal / if not equal, goto .word_not_equal
3973 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3974 leal 4(%eax), %eax / %eax += 4 (next word)
3975 cmpl $4, %edi / if %edi >= 4
3976 jae .word_loop / goto .word_loop
3977 .byte_check:
3978 cmpl $0, %edi / if %edi == 0
3979 je .equal / goto .equal
3980 jmp .byte_loop / goto .byte_loop (checks in bytes)
3981 .word_not_equal:
3982 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3983 .align 4
3984 .byte_loop:
3985 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3986 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3987 jne .not_equal / if not equal, goto .not_equal
3988 incl %ecx / %ecx++ (next byte)
3989 incl %eax / %eax++ (next byte)
3990 decl %edi / %edi--
3991 jnz .byte_loop / if not zero, goto .byte_loop
3992 .equal:
3993 xorl %eax, %eax / %eax = 0
3994 popl %edi / restore register variable
3995 leave / restore old stack frame
3996 ret / return (NULL)
3997 .align 4
3998 .not_equal:
3999 movl $1, %eax / return 1
4000 popl %edi / restore register variable
4001 leave / restore old stack frame
4002 ret / return (NULL)
4003 SET_SIZE(bcmp)
4004
4005 #endif /* __i386 */
4006
4007 #ifdef DEBUG
4008 .text
4009 .bcmp_panic_msg:
4010 .string "bcmp: arguments below kernelbase"
4011 #endif /* DEBUG */
4012
4013 #endif /* __lint */
4014
4015 #if defined(__lint)
4016
4017 uint_t
4018 bsrw_insn(uint16_t mask)
4019 {
4020 uint_t index = sizeof (mask) * NBBY - 1;
4021
4022 while ((mask & (1 << index)) == 0)
4023 index--;
4024 return (index);
4025 }
4026
4027 #else /* __lint */
4028
4029 #if defined(__amd64)
4030
4031 ENTRY_NP(bsrw_insn)
4032 xorl %eax, %eax
4033 bsrw %di, %ax
4034 ret
4035 SET_SIZE(bsrw_insn)
4036
4037 #elif defined(__i386)
4038
4039 ENTRY_NP(bsrw_insn)
4040 movw 4(%esp), %cx
4041 xorl %eax, %eax
4042 bsrw %cx, %ax
4043 ret
4044 SET_SIZE(bsrw_insn)
4045
4046 #endif /* __i386 */
4047 #endif /* __lint */
4048
4049 #if defined(__lint)
4050
4051 uint_t
4052 atomic_btr32(uint32_t *pending, uint_t pil)
4053 {
4054 return (*pending &= ~(1 << pil));
4055 }
4056
4057 #else /* __lint */
4058
4059 #if defined(__i386)
4060
4061 ENTRY_NP(atomic_btr32)
4062 movl 4(%esp), %ecx
4063 movl 8(%esp), %edx
4064 xorl %eax, %eax
4065 lock
4066 btrl %edx, (%ecx)
4067 setc %al
4068 ret
4069 SET_SIZE(atomic_btr32)
4070
4071 #endif /* __i386 */
4072 #endif /* __lint */
4073
4074 #if defined(__lint)
4075
4076 /*ARGSUSED*/
4077 void
4078 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4079 uint_t arg2)
4080 {}
4081
4082 #else /* __lint */
4083
4084 #if defined(__amd64)
4085
4086 ENTRY_NP(switch_sp_and_call)
4087 pushq %rbp
4088 movq %rsp, %rbp /* set up stack frame */
4089 movq %rdi, %rsp /* switch stack pointer */
4090 movq %rdx, %rdi /* pass func arg 1 */
4091 movq %rsi, %r11 /* save function to call */
4092 movq %rcx, %rsi /* pass func arg 2 */
4093 call *%r11 /* call function */
4094 leave /* restore stack */
4095 ret
4096 SET_SIZE(switch_sp_and_call)
4097
4098 #elif defined(__i386)
4099
4100 ENTRY_NP(switch_sp_and_call)
4101 pushl %ebp
4102 mov %esp, %ebp /* set up stack frame */
4103 movl 8(%ebp), %esp /* switch stack pointer */
4104 pushl 20(%ebp) /* push func arg 2 */
4105 pushl 16(%ebp) /* push func arg 1 */
4106 call *12(%ebp) /* call function */
4107 addl $8, %esp /* pop arguments */
4108 leave /* restore stack */
4109 ret
4110 SET_SIZE(switch_sp_and_call)
4111
4112 #endif /* __i386 */
4113 #endif /* __lint */
4114
4115 #if defined(__lint)
4116
4117 void
4118 kmdb_enter(void)
4119 {}
4120
4121 #else /* __lint */
4122
4123 #if defined(__amd64)
4124
4125 ENTRY_NP(kmdb_enter)
4126 pushq %rbp
4127 movq %rsp, %rbp
4128
4129 /*
4130 * Save flags, do a 'cli' then return the saved flags
4131 */
4132 call intr_clear
4133
4134 int $T_DBGENTR
4135
4136 /*
4137 * Restore the saved flags
4138 */
4139 movq %rax, %rdi
4140 call intr_restore
4141
4142 leave
4143 ret
4144 SET_SIZE(kmdb_enter)
4145
4146 #elif defined(__i386)
4147
4148 ENTRY_NP(kmdb_enter)
4149 pushl %ebp
4150 movl %esp, %ebp
4151
4152 /*
4153 * Save flags, do a 'cli' then return the saved flags
4154 */
4155 call intr_clear
4156
4157 int $T_DBGENTR
4158
4159 /*
4160 * Restore the saved flags
4161 */
4162 pushl %eax
4163 call intr_restore
4164 addl $4, %esp
4165
4166 leave
4167 ret
4168 SET_SIZE(kmdb_enter)
4169
4170 #endif /* __i386 */
4171 #endif /* __lint */
4172
4173 #if defined(__lint)
4174
4175 void
4176 return_instr(void)
4177 {}
4178
4179 #else /* __lint */
4180
4181 ENTRY_NP(return_instr)
4182 rep; ret /* use 2 byte instruction when branch target */
4183 /* AMD Software Optimization Guide - Section 6.2 */
4184 SET_SIZE(return_instr)
4185
4186 #endif /* __lint */
4187
4188 #if defined(__lint)
4189
4190 ulong_t
4191 getflags(void)
4192 {
4193 return (0);
4194 }
4195
4196 #else /* __lint */
4197
4198 #if defined(__amd64)
4199
4200 ENTRY(getflags)
4201 pushfq
4202 popq %rax
4203 #if defined(__xpv)
4204 CURTHREAD(%rdi)
4205 KPREEMPT_DISABLE(%rdi)
4206 /*
4207 * Synthesize the PS_IE bit from the event mask bit
4208 */
4209 CURVCPU(%r11)
4210 andq $_BITNOT(PS_IE), %rax
4211 XEN_TEST_UPCALL_MASK(%r11)
4212 jnz 1f
4213 orq $PS_IE, %rax
4214 1:
4215 KPREEMPT_ENABLE_NOKP(%rdi)
4216 #endif
4217 ret
4218 SET_SIZE(getflags)
4219
4220 #elif defined(__i386)
4221
4222 ENTRY(getflags)
4223 pushfl
4224 popl %eax
4225 #if defined(__xpv)
4226 CURTHREAD(%ecx)
4227 KPREEMPT_DISABLE(%ecx)
4228 /*
4229 * Synthesize the PS_IE bit from the event mask bit
4230 */
4231 CURVCPU(%edx)
4232 andl $_BITNOT(PS_IE), %eax
4233 XEN_TEST_UPCALL_MASK(%edx)
4234 jnz 1f
4235 orl $PS_IE, %eax
4236 1:
4237 KPREEMPT_ENABLE_NOKP(%ecx)
4238 #endif
4239 ret
4240 SET_SIZE(getflags)
4241
4242 #endif /* __i386 */
4243
4244 #endif /* __lint */
4245
4246 #if defined(__lint)
4247
4248 ftrace_icookie_t
4249 ftrace_interrupt_disable(void)
4250 { return (0); }
4251
4252 #else /* __lint */
4253
4254 #if defined(__amd64)
4255
4256 ENTRY(ftrace_interrupt_disable)
4257 pushfq
4258 popq %rax
4259 CLI(%rdx)
4260 ret
4261 SET_SIZE(ftrace_interrupt_disable)
4262
4263 #elif defined(__i386)
4264
4265 ENTRY(ftrace_interrupt_disable)
4266 pushfl
4267 popl %eax
4268 CLI(%edx)
4269 ret
4270 SET_SIZE(ftrace_interrupt_disable)
4271
4272 #endif /* __i386 */
4273 #endif /* __lint */
4274
4275 #if defined(__lint)
4276
4277 /*ARGSUSED*/
4278 void
4279 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4280 {}
4281
4282 #else /* __lint */
4283
4284 #if defined(__amd64)
4285
4286 ENTRY(ftrace_interrupt_enable)
4287 pushq %rdi
4288 popfq
4289 ret
4290 SET_SIZE(ftrace_interrupt_enable)
4291
4292 #elif defined(__i386)
4293
4294 ENTRY(ftrace_interrupt_enable)
4295 movl 4(%esp), %eax
4296 pushl %eax
4297 popfl
4298 ret
4299 SET_SIZE(ftrace_interrupt_enable)
4300
4301 #endif /* __i386 */
4302 #endif /* __lint */
4303
4304 #if defined (__lint)
4305
4306 /*ARGSUSED*/
4307 void
4308 clflush_insn(caddr_t addr)
4309 {}
4310
4311 #else /* __lint */
4312
4313 #if defined (__amd64)
4314 ENTRY(clflush_insn)
4315 clflush (%rdi)
4316 ret
4317 SET_SIZE(clflush_insn)
4318 #elif defined (__i386)
4319 ENTRY(clflush_insn)
4320 movl 4(%esp), %eax
4321 clflush (%eax)
4322 ret
4323 SET_SIZE(clflush_insn)
4324
4325 #endif /* __i386 */
4326 #endif /* __lint */
4327
4328 #if defined (__lint)
4329 /*ARGSUSED*/
4330 void
4331 mfence_insn(void)
4332 {}
4333
4334 #else /* __lint */
4335
4336 #if defined (__amd64)
4337 ENTRY(mfence_insn)
4338 mfence
4339 ret
4340 SET_SIZE(mfence_insn)
4341 #elif defined (__i386)
4342 ENTRY(mfence_insn)
4343 mfence
4344 ret
4345 SET_SIZE(mfence_insn)
4346
4347 #endif /* __i386 */
4348 #endif /* __lint */
4349
4350 /*
4351 * VMware implements an I/O port that programs can query to detect if software
4352 * is running in a VMware hypervisor. This hypervisor port behaves differently
4353 * depending on magic values in certain registers and modifies some registers
4354 * as a side effect.
4355 *
4356 * References: http://kb.vmware.com/kb/1009458
4357 */
4358
4359 #if defined(__lint)
4360
4361 /* ARGSUSED */
4362 void
4363 vmware_port(int cmd, uint32_t *regs) { return; }
4364
4365 #else
4366
4367 #if defined(__amd64)
4368
4369 ENTRY(vmware_port)
4370 pushq %rbx
4371 movl $VMWARE_HVMAGIC, %eax
4372 movl $0xffffffff, %ebx
4373 movl %edi, %ecx
4374 movl $VMWARE_HVPORT, %edx
4375 inl (%dx)
4376 movl %eax, (%rsi)
4377 movl %ebx, 4(%rsi)
4378 movl %ecx, 8(%rsi)
4379 movl %edx, 12(%rsi)
4380 popq %rbx
4381 ret
4382 SET_SIZE(vmware_port)
4383
4384 #elif defined(__i386)
4385
4386 ENTRY(vmware_port)
4387 pushl %ebx
4388 pushl %esi
4389 movl $VMWARE_HVMAGIC, %eax
4390 movl $0xffffffff, %ebx
4391 movl 12(%esp), %ecx
4392 movl $VMWARE_HVPORT, %edx
4393 inl (%dx)
4394 movl 16(%esp), %esi
4395 movl %eax, (%esi)
4396 movl %ebx, 4(%esi)
4397 movl %ecx, 8(%esi)
4398 movl %edx, 12(%esi)
4399 popl %esi
4400 popl %ebx
4401 ret
4402 SET_SIZE(vmware_port)
4403
4404 #endif /* __i386 */
4405 #endif /* __lint */