1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 * Copyright 2016 Joyent, Inc.
27 */
28
29 /*
30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 * All Rights Reserved
33 */
34
35 /*
36 * Copyright (c) 2009, Intel Corporation.
37 * All rights reserved.
38 */
39
40 /*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
57 #include <sys/psw.h>
58 #include <sys/x86_archext.h>
59
60 #if defined(__lint)
61 #include <sys/types.h>
62 #include <sys/systm.h>
63 #include <sys/thread.h>
64 #include <sys/archsystm.h>
65 #include <sys/byteorder.h>
66 #include <sys/dtrace.h>
67 #include <sys/ftrace.h>
68 #else /* __lint */
69 #include "assym.h"
70 #endif /* __lint */
71 #include <sys/dditypes.h>
72
73 /*
74 * on_fault()
75 *
76 * Catch lofault faults. Like setjmp except it returns one
77 * if code following causes uncorrectable fault. Turned off
78 * by calling no_fault(). Note that while under on_fault(),
79 * SMAP is disabled. For more information see
80 * uts/intel/ia32/ml/copy.s.
81 */
82
83 #if defined(__lint)
84
85 /* ARGSUSED */
86 int
87 on_fault(label_t *ljb)
88 { return (0); }
89
90 void
91 no_fault(void)
92 {}
93
94 #else /* __lint */
95
96 #if defined(__amd64)
97
98 ENTRY(on_fault)
99 movq %gs:CPU_THREAD, %rsi
100 leaq catch_fault(%rip), %rdx
101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
103 call smap_disable /* allow user accesses */
104 jmp setjmp /* let setjmp do the rest */
105
106 catch_fault:
107 movq %gs:CPU_THREAD, %rsi
108 movq T_ONFAULT(%rsi), %rdi /* address of save area */
109 xorl %eax, %eax
110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
112 call smap_enable /* disallow user accesses */
113 jmp longjmp /* let longjmp do the rest */
114 SET_SIZE(on_fault)
115
116 ENTRY(no_fault)
117 movq %gs:CPU_THREAD, %rsi
118 xorl %eax, %eax
119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
121 call smap_enable /* disallow user accesses */
122 ret
123 SET_SIZE(no_fault)
124
125 #elif defined(__i386)
126
127 ENTRY(on_fault)
128 movl %gs:CPU_THREAD, %edx
129 movl 4(%esp), %eax /* jumpbuf address */
130 leal catch_fault, %ecx
131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
133 jmp setjmp /* let setjmp do the rest */
134
135 catch_fault:
136 movl %gs:CPU_THREAD, %edx
137 xorl %eax, %eax
138 movl T_ONFAULT(%edx), %ecx /* address of save area */
139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
141 pushl %ecx
142 call longjmp /* let longjmp do the rest */
143 SET_SIZE(on_fault)
144
145 ENTRY(no_fault)
146 movl %gs:CPU_THREAD, %edx
147 xorl %eax, %eax
148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
150 ret
151 SET_SIZE(no_fault)
152
153 #endif /* __i386 */
154 #endif /* __lint */
155
156 /*
157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 */
160
161 #if defined(lint)
162
163 void
164 on_trap_trampoline(void)
165 {}
166
167 #else /* __lint */
168
169 #if defined(__amd64)
170
171 ENTRY(on_trap_trampoline)
172 movq %gs:CPU_THREAD, %rsi
173 movq T_ONTRAP(%rsi), %rdi
174 addq $OT_JMPBUF, %rdi
175 jmp longjmp
176 SET_SIZE(on_trap_trampoline)
177
178 #elif defined(__i386)
179
180 ENTRY(on_trap_trampoline)
181 movl %gs:CPU_THREAD, %eax
182 movl T_ONTRAP(%eax), %eax
183 addl $OT_JMPBUF, %eax
184 pushl %eax
185 call longjmp
186 SET_SIZE(on_trap_trampoline)
187
188 #endif /* __i386 */
189 #endif /* __lint */
190
191 /*
192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
193 * more information about the on_trap() mechanism. If the on_trap_data is the
194 * same as the topmost stack element, we just modify that element.
195 */
196 #if defined(lint)
197
198 /*ARGSUSED*/
199 int
200 on_trap(on_trap_data_t *otp, uint_t prot)
201 { return (0); }
202
203 #else /* __lint */
204
205 #if defined(__amd64)
206
207 ENTRY(on_trap)
208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
212 xorl %ecx, %ecx
213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
217 cmpq %rdi, %rcx /* if (otp == %rcx) */
218 je 0f /* don't modify t_ontrap */
219
220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
222
223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
224 jmp setjmp
225 SET_SIZE(on_trap)
226
227 #elif defined(__i386)
228
229 ENTRY(on_trap)
230 movl 4(%esp), %eax /* %eax = otp */
231 movl 8(%esp), %edx /* %edx = prot */
232
233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
241 cmpl %eax, %ecx /* if (otp == %ecx) */
242 je 0f /* don't modify t_ontrap */
243
244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
246
247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
248 movl %eax, 4(%esp) /* put %eax back on the stack */
249 jmp setjmp /* let setjmp do the rest */
250 SET_SIZE(on_trap)
251
252 #endif /* __i386 */
253 #endif /* __lint */
254
255 /*
256 * Setjmp and longjmp implement non-local gotos using state vectors
257 * type label_t.
258 */
259
260 #if defined(__lint)
261
262 /* ARGSUSED */
263 int
264 setjmp(label_t *lp)
265 { return (0); }
266
267 /* ARGSUSED */
268 void
269 longjmp(label_t *lp)
270 {}
271
272 #else /* __lint */
273
274 #if LABEL_PC != 0
275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276 #endif /* LABEL_PC != 0 */
277
278 #if defined(__amd64)
279
280 ENTRY(setjmp)
281 movq %rsp, LABEL_SP(%rdi)
282 movq %rbp, LABEL_RBP(%rdi)
283 movq %rbx, LABEL_RBX(%rdi)
284 movq %r12, LABEL_R12(%rdi)
285 movq %r13, LABEL_R13(%rdi)
286 movq %r14, LABEL_R14(%rdi)
287 movq %r15, LABEL_R15(%rdi)
288 movq (%rsp), %rdx /* return address */
289 movq %rdx, (%rdi) /* LABEL_PC is 0 */
290 xorl %eax, %eax /* return 0 */
291 ret
292 SET_SIZE(setjmp)
293
294 ENTRY(longjmp)
295 movq LABEL_SP(%rdi), %rsp
296 movq LABEL_RBP(%rdi), %rbp
297 movq LABEL_RBX(%rdi), %rbx
298 movq LABEL_R12(%rdi), %r12
299 movq LABEL_R13(%rdi), %r13
300 movq LABEL_R14(%rdi), %r14
301 movq LABEL_R15(%rdi), %r15
302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
303 movq %rdx, (%rsp)
304 xorl %eax, %eax
305 incl %eax /* return 1 */
306 ret
307 SET_SIZE(longjmp)
308
309 #elif defined(__i386)
310
311 ENTRY(setjmp)
312 movl 4(%esp), %edx /* address of save area */
313 movl %ebp, LABEL_EBP(%edx)
314 movl %ebx, LABEL_EBX(%edx)
315 movl %esi, LABEL_ESI(%edx)
316 movl %edi, LABEL_EDI(%edx)
317 movl %esp, 4(%edx)
318 movl (%esp), %ecx /* %eip (return address) */
319 movl %ecx, (%edx) /* LABEL_PC is 0 */
320 subl %eax, %eax /* return 0 */
321 ret
322 SET_SIZE(setjmp)
323
324 ENTRY(longjmp)
325 movl 4(%esp), %edx /* address of save area */
326 movl LABEL_EBP(%edx), %ebp
327 movl LABEL_EBX(%edx), %ebx
328 movl LABEL_ESI(%edx), %esi
329 movl LABEL_EDI(%edx), %edi
330 movl 4(%edx), %esp
331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
332 movl $1, %eax
333 addl $4, %esp /* pop ret adr */
334 jmp *%ecx /* indirect */
335 SET_SIZE(longjmp)
336
337 #endif /* __i386 */
338 #endif /* __lint */
339
340 /*
341 * if a() calls b() calls caller(),
342 * caller() returns return address in a().
343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 * sequence.)
345 */
346
347 #if defined(__lint)
348
349 caddr_t
350 caller(void)
351 { return (0); }
352
353 #else /* __lint */
354
355 #if defined(__amd64)
356
357 ENTRY(caller)
358 movq 8(%rbp), %rax /* b()'s return pc, in a() */
359 ret
360 SET_SIZE(caller)
361
362 #elif defined(__i386)
363
364 ENTRY(caller)
365 movl 4(%ebp), %eax /* b()'s return pc, in a() */
366 ret
367 SET_SIZE(caller)
368
369 #endif /* __i386 */
370 #endif /* __lint */
371
372 /*
373 * if a() calls callee(), callee() returns the
374 * return address in a();
375 */
376
377 #if defined(__lint)
378
379 caddr_t
380 callee(void)
381 { return (0); }
382
383 #else /* __lint */
384
385 #if defined(__amd64)
386
387 ENTRY(callee)
388 movq (%rsp), %rax /* callee()'s return pc, in a() */
389 ret
390 SET_SIZE(callee)
391
392 #elif defined(__i386)
393
394 ENTRY(callee)
395 movl (%esp), %eax /* callee()'s return pc, in a() */
396 ret
397 SET_SIZE(callee)
398
399 #endif /* __i386 */
400 #endif /* __lint */
401
402 /*
403 * return the current frame pointer
404 */
405
406 #if defined(__lint)
407
408 greg_t
409 getfp(void)
410 { return (0); }
411
412 #else /* __lint */
413
414 #if defined(__amd64)
415
416 ENTRY(getfp)
417 movq %rbp, %rax
418 ret
419 SET_SIZE(getfp)
420
421 #elif defined(__i386)
422
423 ENTRY(getfp)
424 movl %ebp, %eax
425 ret
426 SET_SIZE(getfp)
427
428 #endif /* __i386 */
429 #endif /* __lint */
430
431 /*
432 * Invalidate a single page table entry in the TLB
433 */
434
435 #if defined(__lint)
436
437 /* ARGSUSED */
438 void
439 mmu_tlbflush_entry(caddr_t m)
440 {}
441
442 #else /* __lint */
443
444 #if defined(__amd64)
445
446 ENTRY(mmu_tlbflush_entry)
447 invlpg (%rdi)
448 ret
449 SET_SIZE(mmu_tlbflush_entry)
450
451 #elif defined(__i386)
452
453 ENTRY(mmu_tlbflush_entry)
454 movl 4(%esp), %eax
455 invlpg (%eax)
456 ret
457 SET_SIZE(mmu_tlbflush_entry)
458
459 #endif /* __i386 */
460 #endif /* __lint */
461
462
463 /*
464 * Get/Set the value of various control registers
465 */
466
467 #if defined(__lint)
468
469 ulong_t
470 getcr0(void)
471 { return (0); }
472
473 /* ARGSUSED */
474 void
475 setcr0(ulong_t value)
476 {}
477
478 ulong_t
479 getcr2(void)
480 { return (0); }
481
482 ulong_t
483 getcr3(void)
484 { return (0); }
485
486 #if !defined(__xpv)
487 /* ARGSUSED */
488 void
489 setcr3(ulong_t val)
490 {}
491
492 void
493 reload_cr3(void)
494 {}
495 #endif
496
497 ulong_t
498 getcr4(void)
499 { return (0); }
500
501 /* ARGSUSED */
502 void
503 setcr4(ulong_t val)
504 {}
505
506 #if defined(__amd64)
507
508 ulong_t
509 getcr8(void)
510 { return (0); }
511
512 /* ARGSUSED */
513 void
514 setcr8(ulong_t val)
515 {}
516
517 #endif /* __amd64 */
518
519 #else /* __lint */
520
521 #if defined(__amd64)
522
523 ENTRY(getcr0)
524 movq %cr0, %rax
525 ret
526 SET_SIZE(getcr0)
527
528 ENTRY(setcr0)
529 movq %rdi, %cr0
530 ret
531 SET_SIZE(setcr0)
532
533 ENTRY(getcr2)
534 #if defined(__xpv)
535 movq %gs:CPU_VCPU_INFO, %rax
536 movq VCPU_INFO_ARCH_CR2(%rax), %rax
537 #else
538 movq %cr2, %rax
539 #endif
540 ret
541 SET_SIZE(getcr2)
542
543 ENTRY(getcr3)
544 movq %cr3, %rax
545 ret
546 SET_SIZE(getcr3)
547
548 #if !defined(__xpv)
549
550 ENTRY(setcr3)
551 movq %rdi, %cr3
552 ret
553 SET_SIZE(setcr3)
554
555 ENTRY(reload_cr3)
556 movq %cr3, %rdi
557 movq %rdi, %cr3
558 ret
559 SET_SIZE(reload_cr3)
560
561 #endif /* __xpv */
562
563 ENTRY(getcr4)
564 movq %cr4, %rax
565 ret
566 SET_SIZE(getcr4)
567
568 ENTRY(setcr4)
569 movq %rdi, %cr4
570 ret
571 SET_SIZE(setcr4)
572
573 ENTRY(getcr8)
574 movq %cr8, %rax
575 ret
576 SET_SIZE(getcr8)
577
578 ENTRY(setcr8)
579 movq %rdi, %cr8
580 ret
581 SET_SIZE(setcr8)
582
583 #elif defined(__i386)
584
585 ENTRY(getcr0)
586 movl %cr0, %eax
587 ret
588 SET_SIZE(getcr0)
589
590 ENTRY(setcr0)
591 movl 4(%esp), %eax
592 movl %eax, %cr0
593 ret
594 SET_SIZE(setcr0)
595
596 /*
597 * "lock mov %cr0" is used on processors which indicate it is
598 * supported via CPUID. Normally the 32 bit TPR is accessed via
599 * the local APIC.
600 */
601 ENTRY(getcr8)
602 lock
603 movl %cr0, %eax
604 ret
605 SET_SIZE(getcr8)
606
607 ENTRY(setcr8)
608 movl 4(%esp), %eax
609 lock
610 movl %eax, %cr0
611 ret
612 SET_SIZE(setcr8)
613
614 ENTRY(getcr2)
615 #if defined(__xpv)
616 movl %gs:CPU_VCPU_INFO, %eax
617 movl VCPU_INFO_ARCH_CR2(%eax), %eax
618 #else
619 movl %cr2, %eax
620 #endif
621 ret
622 SET_SIZE(getcr2)
623
624 ENTRY(getcr3)
625 movl %cr3, %eax
626 ret
627 SET_SIZE(getcr3)
628
629 #if !defined(__xpv)
630
631 ENTRY(setcr3)
632 movl 4(%esp), %eax
633 movl %eax, %cr3
634 ret
635 SET_SIZE(setcr3)
636
637 ENTRY(reload_cr3)
638 movl %cr3, %eax
639 movl %eax, %cr3
640 ret
641 SET_SIZE(reload_cr3)
642
643 #endif /* __xpv */
644
645 ENTRY(getcr4)
646 movl %cr4, %eax
647 ret
648 SET_SIZE(getcr4)
649
650 ENTRY(setcr4)
651 movl 4(%esp), %eax
652 movl %eax, %cr4
653 ret
654 SET_SIZE(setcr4)
655
656 #endif /* __i386 */
657 #endif /* __lint */
658
659 #if defined(__lint)
660
661 /*ARGSUSED*/
662 uint32_t
663 __cpuid_insn(struct cpuid_regs *regs)
664 { return (0); }
665
666 #else /* __lint */
667
668 #if defined(__amd64)
669
670 ENTRY(__cpuid_insn)
671 movq %rbx, %r8
672 movq %rcx, %r9
673 movq %rdx, %r11
674 movl (%rdi), %eax /* %eax = regs->cp_eax */
675 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
676 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
677 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
678 cpuid
679 movl %eax, (%rdi) /* regs->cp_eax = %eax */
680 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
681 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
682 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
683 movq %r8, %rbx
684 movq %r9, %rcx
685 movq %r11, %rdx
686 ret
687 SET_SIZE(__cpuid_insn)
688
689 #elif defined(__i386)
690
691 ENTRY(__cpuid_insn)
692 pushl %ebp
693 movl 0x8(%esp), %ebp /* %ebp = regs */
694 pushl %ebx
695 pushl %ecx
696 pushl %edx
697 movl (%ebp), %eax /* %eax = regs->cp_eax */
698 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
699 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
700 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
701 cpuid
702 movl %eax, (%ebp) /* regs->cp_eax = %eax */
703 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
704 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
705 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
706 popl %edx
707 popl %ecx
708 popl %ebx
709 popl %ebp
710 ret
711 SET_SIZE(__cpuid_insn)
712
713 #endif /* __i386 */
714 #endif /* __lint */
715
716 #if defined(__lint)
717
718 /*ARGSUSED*/
719 void
720 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
721 {}
722
723 #else /* __lint */
724
725 #if defined(__amd64)
726
727 ENTRY_NP(i86_monitor)
728 pushq %rbp
729 movq %rsp, %rbp
730 movq %rdi, %rax /* addr */
731 movq %rsi, %rcx /* extensions */
732 /* rdx contains input arg3: hints */
733 clflush (%rax)
734 .byte 0x0f, 0x01, 0xc8 /* monitor */
735 leave
736 ret
737 SET_SIZE(i86_monitor)
738
739 #elif defined(__i386)
740
741 ENTRY_NP(i86_monitor)
742 pushl %ebp
743 movl %esp, %ebp
744 movl 0x8(%ebp),%eax /* addr */
745 movl 0xc(%ebp),%ecx /* extensions */
746 movl 0x10(%ebp),%edx /* hints */
747 clflush (%eax)
748 .byte 0x0f, 0x01, 0xc8 /* monitor */
749 leave
750 ret
751 SET_SIZE(i86_monitor)
752
753 #endif /* __i386 */
754 #endif /* __lint */
755
756 #if defined(__lint)
757
758 /*ARGSUSED*/
759 void
760 i86_mwait(uint32_t data, uint32_t extensions)
761 {}
762
763 #else /* __lint */
764
765 #if defined(__amd64)
766
767 ENTRY_NP(i86_mwait)
768 pushq %rbp
769 movq %rsp, %rbp
770 movq %rdi, %rax /* data */
771 movq %rsi, %rcx /* extensions */
772 .byte 0x0f, 0x01, 0xc9 /* mwait */
773 leave
774 ret
775 SET_SIZE(i86_mwait)
776
777 #elif defined(__i386)
778
779 ENTRY_NP(i86_mwait)
780 pushl %ebp
781 movl %esp, %ebp
782 movl 0x8(%ebp),%eax /* data */
783 movl 0xc(%ebp),%ecx /* extensions */
784 .byte 0x0f, 0x01, 0xc9 /* mwait */
785 leave
786 ret
787 SET_SIZE(i86_mwait)
788
789 #endif /* __i386 */
790 #endif /* __lint */
791
792 #if defined(__xpv)
793 /*
794 * Defined in C
795 */
796 #else
797
798 #if defined(__lint)
799
800 hrtime_t
801 tsc_read(void)
802 {
803 return (0);
804 }
805
806 #else /* __lint */
807
808 #if defined(__amd64)
809
810 ENTRY_NP(tsc_read)
811 movq %rbx, %r11
812 movl $0, %eax
813 cpuid
814 rdtsc
815 movq %r11, %rbx
816 shlq $32, %rdx
817 orq %rdx, %rax
818 ret
819 .globl _tsc_mfence_start
820 _tsc_mfence_start:
821 mfence
822 rdtsc
823 shlq $32, %rdx
824 orq %rdx, %rax
825 ret
826 .globl _tsc_mfence_end
827 _tsc_mfence_end:
828 .globl _tscp_start
829 _tscp_start:
830 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
831 shlq $32, %rdx
832 orq %rdx, %rax
833 ret
834 .globl _tscp_end
835 _tscp_end:
836 .globl _no_rdtsc_start
837 _no_rdtsc_start:
838 xorl %edx, %edx
839 xorl %eax, %eax
840 ret
841 .globl _no_rdtsc_end
842 _no_rdtsc_end:
843 .globl _tsc_lfence_start
844 _tsc_lfence_start:
845 lfence
846 rdtsc
847 shlq $32, %rdx
848 orq %rdx, %rax
849 ret
850 .globl _tsc_lfence_end
851 _tsc_lfence_end:
852 SET_SIZE(tsc_read)
853
854 #else /* __i386 */
855
856 ENTRY_NP(tsc_read)
857 pushl %ebx
858 movl $0, %eax
859 cpuid
860 rdtsc
861 popl %ebx
862 ret
863 .globl _tsc_mfence_start
864 _tsc_mfence_start:
865 mfence
866 rdtsc
867 ret
868 .globl _tsc_mfence_end
869 _tsc_mfence_end:
870 .globl _tscp_start
871 _tscp_start:
872 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
873 ret
874 .globl _tscp_end
875 _tscp_end:
876 .globl _no_rdtsc_start
877 _no_rdtsc_start:
878 xorl %edx, %edx
879 xorl %eax, %eax
880 ret
881 .globl _no_rdtsc_end
882 _no_rdtsc_end:
883 .globl _tsc_lfence_start
884 _tsc_lfence_start:
885 lfence
886 rdtsc
887 ret
888 .globl _tsc_lfence_end
889 _tsc_lfence_end:
890 SET_SIZE(tsc_read)
891
892 #endif /* __i386 */
893
894 #endif /* __lint */
895
896
897 #endif /* __xpv */
898
899 #ifdef __lint
900 /*
901 * Do not use this function for obtaining clock tick. This
902 * is called by callers who do not need to have a guarenteed
903 * correct tick value. The proper routine to use is tsc_read().
904 */
905 u_longlong_t
906 randtick(void)
907 {
908 return (0);
909 }
910 #else
911 #if defined(__amd64)
912 ENTRY_NP(randtick)
913 rdtsc
914 shlq $32, %rdx
915 orq %rdx, %rax
916 ret
917 SET_SIZE(randtick)
918 #else
919 ENTRY_NP(randtick)
920 rdtsc
921 ret
922 SET_SIZE(randtick)
923 #endif /* __i386 */
924 #endif /* __lint */
925 /*
926 * Insert entryp after predp in a doubly linked list.
927 */
928
929 #if defined(__lint)
930
931 /*ARGSUSED*/
932 void
933 _insque(caddr_t entryp, caddr_t predp)
934 {}
935
936 #else /* __lint */
937
938 #if defined(__amd64)
939
940 ENTRY(_insque)
941 movq (%rsi), %rax /* predp->forw */
942 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
943 movq %rax, (%rdi) /* entryp->forw = predp->forw */
944 movq %rdi, (%rsi) /* predp->forw = entryp */
945 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
946 ret
947 SET_SIZE(_insque)
948
949 #elif defined(__i386)
950
951 ENTRY(_insque)
952 movl 8(%esp), %edx
953 movl 4(%esp), %ecx
954 movl (%edx), %eax /* predp->forw */
955 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
956 movl %eax, (%ecx) /* entryp->forw = predp->forw */
957 movl %ecx, (%edx) /* predp->forw = entryp */
958 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
959 ret
960 SET_SIZE(_insque)
961
962 #endif /* __i386 */
963 #endif /* __lint */
964
965 /*
966 * Remove entryp from a doubly linked list
967 */
968
969 #if defined(__lint)
970
971 /*ARGSUSED*/
972 void
973 _remque(caddr_t entryp)
974 {}
975
976 #else /* __lint */
977
978 #if defined(__amd64)
979
980 ENTRY(_remque)
981 movq (%rdi), %rax /* entry->forw */
982 movq CPTRSIZE(%rdi), %rdx /* entry->back */
983 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
984 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
985 ret
986 SET_SIZE(_remque)
987
988 #elif defined(__i386)
989
990 ENTRY(_remque)
991 movl 4(%esp), %ecx
992 movl (%ecx), %eax /* entry->forw */
993 movl CPTRSIZE(%ecx), %edx /* entry->back */
994 movl %eax, (%edx) /* entry->back->forw = entry->forw */
995 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
996 ret
997 SET_SIZE(_remque)
998
999 #endif /* __i386 */
1000 #endif /* __lint */
1001
1002 /*
1003 * Returns the number of
1004 * non-NULL bytes in string argument.
1005 */
1006
1007 #if defined(__lint)
1008
1009 /* ARGSUSED */
1010 size_t
1011 strlen(const char *str)
1012 { return (0); }
1013
1014 #else /* __lint */
1015
1016 #if defined(__amd64)
1017
1018 /*
1019 * This is close to a simple transliteration of a C version of this
1020 * routine. We should either just -make- this be a C version, or
1021 * justify having it in assembler by making it significantly faster.
1022 *
1023 * size_t
1024 * strlen(const char *s)
1025 * {
1026 * const char *s0;
1027 * #if defined(DEBUG)
1028 * if ((uintptr_t)s < KERNELBASE)
1029 * panic(.str_panic_msg);
1030 * #endif
1031 * for (s0 = s; *s; s++)
1032 * ;
1033 * return (s - s0);
1034 * }
1035 */
1036
1037 ENTRY(strlen)
1038 #ifdef DEBUG
1039 movq postbootkernelbase(%rip), %rax
1040 cmpq %rax, %rdi
1041 jae str_valid
1042 pushq %rbp
1043 movq %rsp, %rbp
1044 leaq .str_panic_msg(%rip), %rdi
1045 xorl %eax, %eax
1046 call panic
1047 #endif /* DEBUG */
1048 str_valid:
1049 cmpb $0, (%rdi)
1050 movq %rdi, %rax
1051 je .null_found
1052 .align 4
1053 .strlen_loop:
1054 incq %rdi
1055 cmpb $0, (%rdi)
1056 jne .strlen_loop
1057 .null_found:
1058 subq %rax, %rdi
1059 movq %rdi, %rax
1060 ret
1061 SET_SIZE(strlen)
1062
1063 #elif defined(__i386)
1064
1065 ENTRY(strlen)
1066 #ifdef DEBUG
1067 movl postbootkernelbase, %eax
1068 cmpl %eax, 4(%esp)
1069 jae str_valid
1070 pushl %ebp
1071 movl %esp, %ebp
1072 pushl $.str_panic_msg
1073 call panic
1074 #endif /* DEBUG */
1075
1076 str_valid:
1077 movl 4(%esp), %eax /* %eax = string address */
1078 testl $3, %eax /* if %eax not word aligned */
1079 jnz .not_word_aligned /* goto .not_word_aligned */
1080 .align 4
1081 .word_aligned:
1082 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1083 movl $0x7f7f7f7f, %ecx
1084 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1085 addl $4, %eax /* next word */
1086 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1087 orl %edx, %ecx /* %ecx |= %edx */
1088 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1089 cmpl $0x80808080, %ecx /* if no null byte in this word */
1090 je .word_aligned /* goto .word_aligned */
1091 subl $4, %eax /* post-incremented */
1092 .not_word_aligned:
1093 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1094 je .null_found /* goto .null_found */
1095 incl %eax /* next byte */
1096 testl $3, %eax /* if %eax not word aligned */
1097 jnz .not_word_aligned /* goto .not_word_aligned */
1098 jmp .word_aligned /* goto .word_aligned */
1099 .align 4
1100 .null_found:
1101 subl 4(%esp), %eax /* %eax -= string address */
1102 ret
1103 SET_SIZE(strlen)
1104
1105 #endif /* __i386 */
1106
1107 #ifdef DEBUG
1108 .text
1109 .str_panic_msg:
1110 .string "strlen: argument below kernelbase"
1111 #endif /* DEBUG */
1112
1113 #endif /* __lint */
1114
1115 /*
1116 * Berkeley 4.3 introduced symbolically named interrupt levels
1117 * as a way deal with priority in a machine independent fashion.
1118 * Numbered priorities are machine specific, and should be
1119 * discouraged where possible.
1120 *
1121 * Note, for the machine specific priorities there are
1122 * examples listed for devices that use a particular priority.
1123 * It should not be construed that all devices of that
1124 * type should be at that priority. It is currently were
1125 * the current devices fit into the priority scheme based
1126 * upon time criticalness.
1127 *
1128 * The underlying assumption of these assignments is that
1129 * IPL 10 is the highest level from which a device
1130 * routine can call wakeup. Devices that interrupt from higher
1131 * levels are restricted in what they can do. If they need
1132 * kernels services they should schedule a routine at a lower
1133 * level (via software interrupt) to do the required
1134 * processing.
1135 *
1136 * Examples of this higher usage:
1137 * Level Usage
1138 * 14 Profiling clock (and PROM uart polling clock)
1139 * 12 Serial ports
1140 *
1141 * The serial ports request lower level processing on level 6.
1142 *
1143 * Also, almost all splN routines (where N is a number or a
1144 * mnemonic) will do a RAISE(), on the assumption that they are
1145 * never used to lower our priority.
1146 * The exceptions are:
1147 * spl8() Because you can't be above 15 to begin with!
1148 * splzs() Because this is used at boot time to lower our
1149 * priority, to allow the PROM to poll the uart.
1150 * spl0() Used to lower priority to 0.
1151 */
1152
1153 #if defined(__lint)
1154
1155 int spl0(void) { return (0); }
1156 int spl6(void) { return (0); }
1157 int spl7(void) { return (0); }
1158 int spl8(void) { return (0); }
1159 int splhigh(void) { return (0); }
1160 int splhi(void) { return (0); }
1161 int splzs(void) { return (0); }
1162
1163 /* ARGSUSED */
1164 void
1165 splx(int level)
1166 {}
1167
1168 #else /* __lint */
1169
1170 #if defined(__amd64)
1171
1172 #define SETPRI(level) \
1173 movl $/**/level, %edi; /* new priority */ \
1174 jmp do_splx /* redirect to do_splx */
1175
1176 #define RAISE(level) \
1177 movl $/**/level, %edi; /* new priority */ \
1178 jmp splr /* redirect to splr */
1179
1180 #elif defined(__i386)
1181
1182 #define SETPRI(level) \
1183 pushl $/**/level; /* new priority */ \
1184 call do_splx; /* invoke common splx code */ \
1185 addl $4, %esp; /* unstack arg */ \
1186 ret
1187
1188 #define RAISE(level) \
1189 pushl $/**/level; /* new priority */ \
1190 call splr; /* invoke common splr code */ \
1191 addl $4, %esp; /* unstack args */ \
1192 ret
1193
1194 #endif /* __i386 */
1195
1196 /* locks out all interrupts, including memory errors */
1197 ENTRY(spl8)
1198 SETPRI(15)
1199 SET_SIZE(spl8)
1200
1201 /* just below the level that profiling runs */
1202 ENTRY(spl7)
1203 RAISE(13)
1204 SET_SIZE(spl7)
1205
1206 /* sun specific - highest priority onboard serial i/o asy ports */
1207 ENTRY(splzs)
1208 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1209 SET_SIZE(splzs)
1210
1211 ENTRY(splhi)
1212 ALTENTRY(splhigh)
1213 ALTENTRY(spl6)
1214 ALTENTRY(i_ddi_splhigh)
1215
1216 RAISE(DISP_LEVEL)
1217
1218 SET_SIZE(i_ddi_splhigh)
1219 SET_SIZE(spl6)
1220 SET_SIZE(splhigh)
1221 SET_SIZE(splhi)
1222
1223 /* allow all interrupts */
1224 ENTRY(spl0)
1225 SETPRI(0)
1226 SET_SIZE(spl0)
1227
1228
1229 /* splx implementation */
1230 ENTRY(splx)
1231 jmp do_splx /* redirect to common splx code */
1232 SET_SIZE(splx)
1233
1234 #endif /* __lint */
1235
1236 #if defined(__i386)
1237
1238 /*
1239 * Read and write the %gs register
1240 */
1241
1242 #if defined(__lint)
1243
1244 /*ARGSUSED*/
1245 uint16_t
1246 getgs(void)
1247 { return (0); }
1248
1249 /*ARGSUSED*/
1250 void
1251 setgs(uint16_t sel)
1252 {}
1253
1254 #else /* __lint */
1255
1256 ENTRY(getgs)
1257 clr %eax
1258 movw %gs, %ax
1259 ret
1260 SET_SIZE(getgs)
1261
1262 ENTRY(setgs)
1263 movw 4(%esp), %gs
1264 ret
1265 SET_SIZE(setgs)
1266
1267 #endif /* __lint */
1268 #endif /* __i386 */
1269
1270 #if defined(__lint)
1271
1272 void
1273 pc_reset(void)
1274 {}
1275
1276 void
1277 efi_reset(void)
1278 {}
1279
1280 #else /* __lint */
1281
1282 ENTRY(wait_500ms)
1283 #if defined(__amd64)
1284 pushq %rbx
1285 #elif defined(__i386)
1286 push %ebx
1287 #endif
1288 movl $50000, %ebx
1289 1:
1290 call tenmicrosec
1291 decl %ebx
1292 jnz 1b
1293 #if defined(__amd64)
1294 popq %rbx
1295 #elif defined(__i386)
1296 pop %ebx
1297 #endif
1298 ret
1299 SET_SIZE(wait_500ms)
1300
1301 #define RESET_METHOD_KBC 1
1302 #define RESET_METHOD_PORT92 2
1303 #define RESET_METHOD_PCI 4
1304
1305 DGDEF3(pc_reset_methods, 4, 8)
1306 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1307
1308 ENTRY(pc_reset)
1309
1310 #if defined(__i386)
1311 testl $RESET_METHOD_KBC, pc_reset_methods
1312 #elif defined(__amd64)
1313 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1314 #endif
1315 jz 1f
1316
1317 /
1318 / Try the classic keyboard controller-triggered reset.
1319 /
1320 movw $0x64, %dx
1321 movb $0xfe, %al
1322 outb (%dx)
1323
1324 / Wait up to 500 milliseconds here for the keyboard controller
1325 / to pull the reset line. On some systems where the keyboard
1326 / controller is slow to pull the reset line, the next reset method
1327 / may be executed (which may be bad if those systems hang when the
1328 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1329 / and Ferrari 4000 (doesn't like the cf9 reset method))
1330
1331 call wait_500ms
1332
1333 1:
1334 #if defined(__i386)
1335 testl $RESET_METHOD_PORT92, pc_reset_methods
1336 #elif defined(__amd64)
1337 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1338 #endif
1339 jz 3f
1340
1341 /
1342 / Try port 0x92 fast reset
1343 /
1344 movw $0x92, %dx
1345 inb (%dx)
1346 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1347 je 1f
1348 testb $1, %al / If bit 0
1349 jz 2f / is clear, jump to perform the reset
1350 andb $0xfe, %al / otherwise,
1351 outb (%dx) / clear bit 0 first, then
1352 2:
1353 orb $1, %al / Set bit 0
1354 outb (%dx) / and reset the system
1355 1:
1356
1357 call wait_500ms
1358
1359 3:
1360 #if defined(__i386)
1361 testl $RESET_METHOD_PCI, pc_reset_methods
1362 #elif defined(__amd64)
1363 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1364 #endif
1365 jz 4f
1366
1367 / Try the PCI (soft) reset vector (should work on all modern systems,
1368 / but has been shown to cause problems on 450NX systems, and some newer
1369 / systems (e.g. ATI IXP400-equipped systems))
1370 / When resetting via this method, 2 writes are required. The first
1371 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1372 / power cycle).
1373 / The reset occurs on the second write, during bit 2's transition from
1374 / 0->1.
1375 movw $0xcf9, %dx
1376 movb $0x2, %al / Reset mode = hard, no power cycle
1377 outb (%dx)
1378 movb $0x6, %al
1379 outb (%dx)
1380
1381 call wait_500ms
1382
1383 4:
1384 /
1385 / port 0xcf9 failed also. Last-ditch effort is to
1386 / triple-fault the CPU.
1387 / Also, use triple fault for EFI firmware
1388 /
1389 ENTRY(efi_reset)
1390 #if defined(__amd64)
1391 pushq $0x0
1392 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1393 lidt (%rsp)
1394 #elif defined(__i386)
1395 pushl $0x0
1396 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1397 lidt (%esp)
1398 #endif
1399 int $0x0 / Trigger interrupt, generate triple-fault
1400
1401 cli
1402 hlt / Wait forever
1403 /*NOTREACHED*/
1404 SET_SIZE(efi_reset)
1405 SET_SIZE(pc_reset)
1406
1407 #endif /* __lint */
1408
1409 /*
1410 * C callable in and out routines
1411 */
1412
1413 #if defined(__lint)
1414
1415 /* ARGSUSED */
1416 void
1417 outl(int port_address, uint32_t val)
1418 {}
1419
1420 #else /* __lint */
1421
1422 #if defined(__amd64)
1423
1424 ENTRY(outl)
1425 movw %di, %dx
1426 movl %esi, %eax
1427 outl (%dx)
1428 ret
1429 SET_SIZE(outl)
1430
1431 #elif defined(__i386)
1432
1433 .set PORT, 4
1434 .set VAL, 8
1435
1436 ENTRY(outl)
1437 movw PORT(%esp), %dx
1438 movl VAL(%esp), %eax
1439 outl (%dx)
1440 ret
1441 SET_SIZE(outl)
1442
1443 #endif /* __i386 */
1444 #endif /* __lint */
1445
1446 #if defined(__lint)
1447
1448 /* ARGSUSED */
1449 void
1450 outw(int port_address, uint16_t val)
1451 {}
1452
1453 #else /* __lint */
1454
1455 #if defined(__amd64)
1456
1457 ENTRY(outw)
1458 movw %di, %dx
1459 movw %si, %ax
1460 D16 outl (%dx) /* XX64 why not outw? */
1461 ret
1462 SET_SIZE(outw)
1463
1464 #elif defined(__i386)
1465
1466 ENTRY(outw)
1467 movw PORT(%esp), %dx
1468 movw VAL(%esp), %ax
1469 D16 outl (%dx)
1470 ret
1471 SET_SIZE(outw)
1472
1473 #endif /* __i386 */
1474 #endif /* __lint */
1475
1476 #if defined(__lint)
1477
1478 /* ARGSUSED */
1479 void
1480 outb(int port_address, uint8_t val)
1481 {}
1482
1483 #else /* __lint */
1484
1485 #if defined(__amd64)
1486
1487 ENTRY(outb)
1488 movw %di, %dx
1489 movb %sil, %al
1490 outb (%dx)
1491 ret
1492 SET_SIZE(outb)
1493
1494 #elif defined(__i386)
1495
1496 ENTRY(outb)
1497 movw PORT(%esp), %dx
1498 movb VAL(%esp), %al
1499 outb (%dx)
1500 ret
1501 SET_SIZE(outb)
1502
1503 #endif /* __i386 */
1504 #endif /* __lint */
1505
1506 #if defined(__lint)
1507
1508 /* ARGSUSED */
1509 uint32_t
1510 inl(int port_address)
1511 { return (0); }
1512
1513 #else /* __lint */
1514
1515 #if defined(__amd64)
1516
1517 ENTRY(inl)
1518 xorl %eax, %eax
1519 movw %di, %dx
1520 inl (%dx)
1521 ret
1522 SET_SIZE(inl)
1523
1524 #elif defined(__i386)
1525
1526 ENTRY(inl)
1527 movw PORT(%esp), %dx
1528 inl (%dx)
1529 ret
1530 SET_SIZE(inl)
1531
1532 #endif /* __i386 */
1533 #endif /* __lint */
1534
1535 #if defined(__lint)
1536
1537 /* ARGSUSED */
1538 uint16_t
1539 inw(int port_address)
1540 { return (0); }
1541
1542 #else /* __lint */
1543
1544 #if defined(__amd64)
1545
1546 ENTRY(inw)
1547 xorl %eax, %eax
1548 movw %di, %dx
1549 D16 inl (%dx)
1550 ret
1551 SET_SIZE(inw)
1552
1553 #elif defined(__i386)
1554
1555 ENTRY(inw)
1556 subl %eax, %eax
1557 movw PORT(%esp), %dx
1558 D16 inl (%dx)
1559 ret
1560 SET_SIZE(inw)
1561
1562 #endif /* __i386 */
1563 #endif /* __lint */
1564
1565
1566 #if defined(__lint)
1567
1568 /* ARGSUSED */
1569 uint8_t
1570 inb(int port_address)
1571 { return (0); }
1572
1573 #else /* __lint */
1574
1575 #if defined(__amd64)
1576
1577 ENTRY(inb)
1578 xorl %eax, %eax
1579 movw %di, %dx
1580 inb (%dx)
1581 ret
1582 SET_SIZE(inb)
1583
1584 #elif defined(__i386)
1585
1586 ENTRY(inb)
1587 subl %eax, %eax
1588 movw PORT(%esp), %dx
1589 inb (%dx)
1590 ret
1591 SET_SIZE(inb)
1592
1593 #endif /* __i386 */
1594 #endif /* __lint */
1595
1596
1597 #if defined(__lint)
1598
1599 /* ARGSUSED */
1600 void
1601 repoutsw(int port, uint16_t *addr, int cnt)
1602 {}
1603
1604 #else /* __lint */
1605
1606 #if defined(__amd64)
1607
1608 ENTRY(repoutsw)
1609 movl %edx, %ecx
1610 movw %di, %dx
1611 rep
1612 D16 outsl
1613 ret
1614 SET_SIZE(repoutsw)
1615
1616 #elif defined(__i386)
1617
1618 /*
1619 * The arguments and saved registers are on the stack in the
1620 * following order:
1621 * | cnt | +16
1622 * | *addr | +12
1623 * | port | +8
1624 * | eip | +4
1625 * | esi | <-- %esp
1626 * If additional values are pushed onto the stack, make sure
1627 * to adjust the following constants accordingly.
1628 */
1629 .set PORT, 8
1630 .set ADDR, 12
1631 .set COUNT, 16
1632
1633 ENTRY(repoutsw)
1634 pushl %esi
1635 movl PORT(%esp), %edx
1636 movl ADDR(%esp), %esi
1637 movl COUNT(%esp), %ecx
1638 rep
1639 D16 outsl
1640 popl %esi
1641 ret
1642 SET_SIZE(repoutsw)
1643
1644 #endif /* __i386 */
1645 #endif /* __lint */
1646
1647
1648 #if defined(__lint)
1649
1650 /* ARGSUSED */
1651 void
1652 repinsw(int port_addr, uint16_t *addr, int cnt)
1653 {}
1654
1655 #else /* __lint */
1656
1657 #if defined(__amd64)
1658
1659 ENTRY(repinsw)
1660 movl %edx, %ecx
1661 movw %di, %dx
1662 rep
1663 D16 insl
1664 ret
1665 SET_SIZE(repinsw)
1666
1667 #elif defined(__i386)
1668
1669 ENTRY(repinsw)
1670 pushl %edi
1671 movl PORT(%esp), %edx
1672 movl ADDR(%esp), %edi
1673 movl COUNT(%esp), %ecx
1674 rep
1675 D16 insl
1676 popl %edi
1677 ret
1678 SET_SIZE(repinsw)
1679
1680 #endif /* __i386 */
1681 #endif /* __lint */
1682
1683
1684 #if defined(__lint)
1685
1686 /* ARGSUSED */
1687 void
1688 repinsb(int port, uint8_t *addr, int count)
1689 {}
1690
1691 #else /* __lint */
1692
1693 #if defined(__amd64)
1694
1695 ENTRY(repinsb)
1696 movl %edx, %ecx
1697 movw %di, %dx
1698 movq %rsi, %rdi
1699 rep
1700 insb
1701 ret
1702 SET_SIZE(repinsb)
1703
1704 #elif defined(__i386)
1705
1706 /*
1707 * The arguments and saved registers are on the stack in the
1708 * following order:
1709 * | cnt | +16
1710 * | *addr | +12
1711 * | port | +8
1712 * | eip | +4
1713 * | esi | <-- %esp
1714 * If additional values are pushed onto the stack, make sure
1715 * to adjust the following constants accordingly.
1716 */
1717 .set IO_PORT, 8
1718 .set IO_ADDR, 12
1719 .set IO_COUNT, 16
1720
1721 ENTRY(repinsb)
1722 pushl %edi
1723 movl IO_ADDR(%esp), %edi
1724 movl IO_COUNT(%esp), %ecx
1725 movl IO_PORT(%esp), %edx
1726 rep
1727 insb
1728 popl %edi
1729 ret
1730 SET_SIZE(repinsb)
1731
1732 #endif /* __i386 */
1733 #endif /* __lint */
1734
1735
1736 /*
1737 * Input a stream of 32-bit words.
1738 * NOTE: count is a DWORD count.
1739 */
1740 #if defined(__lint)
1741
1742 /* ARGSUSED */
1743 void
1744 repinsd(int port, uint32_t *addr, int count)
1745 {}
1746
1747 #else /* __lint */
1748
1749 #if defined(__amd64)
1750
1751 ENTRY(repinsd)
1752 movl %edx, %ecx
1753 movw %di, %dx
1754 movq %rsi, %rdi
1755 rep
1756 insl
1757 ret
1758 SET_SIZE(repinsd)
1759
1760 #elif defined(__i386)
1761
1762 ENTRY(repinsd)
1763 pushl %edi
1764 movl IO_ADDR(%esp), %edi
1765 movl IO_COUNT(%esp), %ecx
1766 movl IO_PORT(%esp), %edx
1767 rep
1768 insl
1769 popl %edi
1770 ret
1771 SET_SIZE(repinsd)
1772
1773 #endif /* __i386 */
1774 #endif /* __lint */
1775
1776 /*
1777 * Output a stream of bytes
1778 * NOTE: count is a byte count
1779 */
1780 #if defined(__lint)
1781
1782 /* ARGSUSED */
1783 void
1784 repoutsb(int port, uint8_t *addr, int count)
1785 {}
1786
1787 #else /* __lint */
1788
1789 #if defined(__amd64)
1790
1791 ENTRY(repoutsb)
1792 movl %edx, %ecx
1793 movw %di, %dx
1794 rep
1795 outsb
1796 ret
1797 SET_SIZE(repoutsb)
1798
1799 #elif defined(__i386)
1800
1801 ENTRY(repoutsb)
1802 pushl %esi
1803 movl IO_ADDR(%esp), %esi
1804 movl IO_COUNT(%esp), %ecx
1805 movl IO_PORT(%esp), %edx
1806 rep
1807 outsb
1808 popl %esi
1809 ret
1810 SET_SIZE(repoutsb)
1811
1812 #endif /* __i386 */
1813 #endif /* __lint */
1814
1815 /*
1816 * Output a stream of 32-bit words
1817 * NOTE: count is a DWORD count
1818 */
1819 #if defined(__lint)
1820
1821 /* ARGSUSED */
1822 void
1823 repoutsd(int port, uint32_t *addr, int count)
1824 {}
1825
1826 #else /* __lint */
1827
1828 #if defined(__amd64)
1829
1830 ENTRY(repoutsd)
1831 movl %edx, %ecx
1832 movw %di, %dx
1833 rep
1834 outsl
1835 ret
1836 SET_SIZE(repoutsd)
1837
1838 #elif defined(__i386)
1839
1840 ENTRY(repoutsd)
1841 pushl %esi
1842 movl IO_ADDR(%esp), %esi
1843 movl IO_COUNT(%esp), %ecx
1844 movl IO_PORT(%esp), %edx
1845 rep
1846 outsl
1847 popl %esi
1848 ret
1849 SET_SIZE(repoutsd)
1850
1851 #endif /* __i386 */
1852 #endif /* __lint */
1853
1854 /*
1855 * void int3(void)
1856 * void int18(void)
1857 * void int20(void)
1858 * void int_cmci(void)
1859 */
1860
1861 #if defined(__lint)
1862
1863 void
1864 int3(void)
1865 {}
1866
1867 void
1868 int18(void)
1869 {}
1870
1871 void
1872 int20(void)
1873 {}
1874
1875 void
1876 int_cmci(void)
1877 {}
1878
1879 #else /* __lint */
1880
1881 ENTRY(int3)
1882 int $T_BPTFLT
1883 ret
1884 SET_SIZE(int3)
1885
1886 ENTRY(int18)
1887 int $T_MCE
1888 ret
1889 SET_SIZE(int18)
1890
1891 ENTRY(int20)
1892 movl boothowto, %eax
1893 andl $RB_DEBUG, %eax
1894 jz 1f
1895
1896 int $T_DBGENTR
1897 1:
1898 rep; ret /* use 2 byte return instruction when branch target */
1899 /* AMD Software Optimization Guide - Section 6.2 */
1900 SET_SIZE(int20)
1901
1902 ENTRY(int_cmci)
1903 int $T_ENOEXTFLT
1904 ret
1905 SET_SIZE(int_cmci)
1906
1907 #endif /* __lint */
1908
1909 #if defined(__lint)
1910
1911 /* ARGSUSED */
1912 int
1913 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1914 { return (0); }
1915
1916 #else /* __lint */
1917
1918 #if defined(__amd64)
1919
1920 ENTRY(scanc)
1921 /* rdi == size */
1922 /* rsi == cp */
1923 /* rdx == table */
1924 /* rcx == mask */
1925 addq %rsi, %rdi /* end = &cp[size] */
1926 .scanloop:
1927 cmpq %rdi, %rsi /* while (cp < end */
1928 jnb .scandone
1929 movzbq (%rsi), %r8 /* %r8 = *cp */
1930 incq %rsi /* cp++ */
1931 testb %cl, (%r8, %rdx)
1932 jz .scanloop /* && (table[*cp] & mask) == 0) */
1933 decq %rsi /* (fix post-increment) */
1934 .scandone:
1935 movl %edi, %eax
1936 subl %esi, %eax /* return (end - cp) */
1937 ret
1938 SET_SIZE(scanc)
1939
1940 #elif defined(__i386)
1941
1942 ENTRY(scanc)
1943 pushl %edi
1944 pushl %esi
1945 movb 24(%esp), %cl /* mask = %cl */
1946 movl 16(%esp), %esi /* cp = %esi */
1947 movl 20(%esp), %edx /* table = %edx */
1948 movl %esi, %edi
1949 addl 12(%esp), %edi /* end = &cp[size]; */
1950 .scanloop:
1951 cmpl %edi, %esi /* while (cp < end */
1952 jnb .scandone
1953 movzbl (%esi), %eax /* %al = *cp */
1954 incl %esi /* cp++ */
1955 movb (%edx, %eax), %al /* %al = table[*cp] */
1956 testb %al, %cl
1957 jz .scanloop /* && (table[*cp] & mask) == 0) */
1958 dec %esi /* post-incremented */
1959 .scandone:
1960 movl %edi, %eax
1961 subl %esi, %eax /* return (end - cp) */
1962 popl %esi
1963 popl %edi
1964 ret
1965 SET_SIZE(scanc)
1966
1967 #endif /* __i386 */
1968 #endif /* __lint */
1969
1970 /*
1971 * Replacement functions for ones that are normally inlined.
1972 * In addition to the copy in i86.il, they are defined here just in case.
1973 */
1974
1975 #if defined(__lint)
1976
1977 ulong_t
1978 intr_clear(void)
1979 { return (0); }
1980
1981 ulong_t
1982 clear_int_flag(void)
1983 { return (0); }
1984
1985 #else /* __lint */
1986
1987 #if defined(__amd64)
1988
1989 ENTRY(intr_clear)
1990 ENTRY(clear_int_flag)
1991 pushfq
1992 popq %rax
1993 #if defined(__xpv)
1994 leaq xpv_panicking, %rdi
1995 movl (%rdi), %edi
1996 cmpl $0, %edi
1997 jne 2f
1998 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1999 /*
2000 * Synthesize the PS_IE bit from the event mask bit
2001 */
2002 andq $_BITNOT(PS_IE), %rax
2003 testb $1, %dl
2004 jnz 1f
2005 orq $PS_IE, %rax
2006 1:
2007 ret
2008 2:
2009 #endif
2010 CLI(%rdi)
2011 ret
2012 SET_SIZE(clear_int_flag)
2013 SET_SIZE(intr_clear)
2014
2015 #elif defined(__i386)
2016
2017 ENTRY(intr_clear)
2018 ENTRY(clear_int_flag)
2019 pushfl
2020 popl %eax
2021 #if defined(__xpv)
2022 leal xpv_panicking, %edx
2023 movl (%edx), %edx
2024 cmpl $0, %edx
2025 jne 2f
2026 CLIRET(%edx, %cl) /* returns event mask in %cl */
2027 /*
2028 * Synthesize the PS_IE bit from the event mask bit
2029 */
2030 andl $_BITNOT(PS_IE), %eax
2031 testb $1, %cl
2032 jnz 1f
2033 orl $PS_IE, %eax
2034 1:
2035 ret
2036 2:
2037 #endif
2038 CLI(%edx)
2039 ret
2040 SET_SIZE(clear_int_flag)
2041 SET_SIZE(intr_clear)
2042
2043 #endif /* __i386 */
2044 #endif /* __lint */
2045
2046 #if defined(__lint)
2047
2048 struct cpu *
2049 curcpup(void)
2050 { return 0; }
2051
2052 #else /* __lint */
2053
2054 #if defined(__amd64)
2055
2056 ENTRY(curcpup)
2057 movq %gs:CPU_SELF, %rax
2058 ret
2059 SET_SIZE(curcpup)
2060
2061 #elif defined(__i386)
2062
2063 ENTRY(curcpup)
2064 movl %gs:CPU_SELF, %eax
2065 ret
2066 SET_SIZE(curcpup)
2067
2068 #endif /* __i386 */
2069 #endif /* __lint */
2070
2071 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2072 * These functions reverse the byte order of the input parameter and returns
2073 * the result. This is to convert the byte order from host byte order
2074 * (little endian) to network byte order (big endian), or vice versa.
2075 */
2076
2077 #if defined(__lint)
2078
2079 uint64_t
2080 htonll(uint64_t i)
2081 { return (i); }
2082
2083 uint64_t
2084 ntohll(uint64_t i)
2085 { return (i); }
2086
2087 uint32_t
2088 htonl(uint32_t i)
2089 { return (i); }
2090
2091 uint32_t
2092 ntohl(uint32_t i)
2093 { return (i); }
2094
2095 uint16_t
2096 htons(uint16_t i)
2097 { return (i); }
2098
2099 uint16_t
2100 ntohs(uint16_t i)
2101 { return (i); }
2102
2103 #else /* __lint */
2104
2105 #if defined(__amd64)
2106
2107 ENTRY(htonll)
2108 ALTENTRY(ntohll)
2109 movq %rdi, %rax
2110 bswapq %rax
2111 ret
2112 SET_SIZE(ntohll)
2113 SET_SIZE(htonll)
2114
2115 /* XX64 there must be shorter sequences for this */
2116 ENTRY(htonl)
2117 ALTENTRY(ntohl)
2118 movl %edi, %eax
2119 bswap %eax
2120 ret
2121 SET_SIZE(ntohl)
2122 SET_SIZE(htonl)
2123
2124 /* XX64 there must be better sequences for this */
2125 ENTRY(htons)
2126 ALTENTRY(ntohs)
2127 movl %edi, %eax
2128 bswap %eax
2129 shrl $16, %eax
2130 ret
2131 SET_SIZE(ntohs)
2132 SET_SIZE(htons)
2133
2134 #elif defined(__i386)
2135
2136 ENTRY(htonll)
2137 ALTENTRY(ntohll)
2138 movl 4(%esp), %edx
2139 movl 8(%esp), %eax
2140 bswap %edx
2141 bswap %eax
2142 ret
2143 SET_SIZE(ntohll)
2144 SET_SIZE(htonll)
2145
2146 ENTRY(htonl)
2147 ALTENTRY(ntohl)
2148 movl 4(%esp), %eax
2149 bswap %eax
2150 ret
2151 SET_SIZE(ntohl)
2152 SET_SIZE(htonl)
2153
2154 ENTRY(htons)
2155 ALTENTRY(ntohs)
2156 movl 4(%esp), %eax
2157 bswap %eax
2158 shrl $16, %eax
2159 ret
2160 SET_SIZE(ntohs)
2161 SET_SIZE(htons)
2162
2163 #endif /* __i386 */
2164 #endif /* __lint */
2165
2166
2167 #if defined(__lint)
2168
2169 /* ARGSUSED */
2170 void
2171 intr_restore(ulong_t i)
2172 { return; }
2173
2174 /* ARGSUSED */
2175 void
2176 restore_int_flag(ulong_t i)
2177 { return; }
2178
2179 #else /* __lint */
2180
2181 #if defined(__amd64)
2182
2183 ENTRY(intr_restore)
2184 ENTRY(restore_int_flag)
2185 testq $PS_IE, %rdi
2186 jz 1f
2187 #if defined(__xpv)
2188 leaq xpv_panicking, %rsi
2189 movl (%rsi), %esi
2190 cmpl $0, %esi
2191 jne 1f
2192 /*
2193 * Since we're -really- running unprivileged, our attempt
2194 * to change the state of the IF bit will be ignored.
2195 * The virtual IF bit is tweaked by CLI and STI.
2196 */
2197 IE_TO_EVENT_MASK(%rsi, %rdi)
2198 #else
2199 sti
2200 #endif
2201 1:
2202 ret
2203 SET_SIZE(restore_int_flag)
2204 SET_SIZE(intr_restore)
2205
2206 #elif defined(__i386)
2207
2208 ENTRY(intr_restore)
2209 ENTRY(restore_int_flag)
2210 testl $PS_IE, 4(%esp)
2211 jz 1f
2212 #if defined(__xpv)
2213 leal xpv_panicking, %edx
2214 movl (%edx), %edx
2215 cmpl $0, %edx
2216 jne 1f
2217 /*
2218 * Since we're -really- running unprivileged, our attempt
2219 * to change the state of the IF bit will be ignored.
2220 * The virtual IF bit is tweaked by CLI and STI.
2221 */
2222 IE_TO_EVENT_MASK(%edx, 4(%esp))
2223 #else
2224 sti
2225 #endif
2226 1:
2227 ret
2228 SET_SIZE(restore_int_flag)
2229 SET_SIZE(intr_restore)
2230
2231 #endif /* __i386 */
2232 #endif /* __lint */
2233
2234 #if defined(__lint)
2235
2236 void
2237 sti(void)
2238 {}
2239
2240 void
2241 cli(void)
2242 {}
2243
2244 #else /* __lint */
2245
2246 ENTRY(sti)
2247 STI
2248 ret
2249 SET_SIZE(sti)
2250
2251 ENTRY(cli)
2252 #if defined(__amd64)
2253 CLI(%rax)
2254 #elif defined(__i386)
2255 CLI(%eax)
2256 #endif /* __i386 */
2257 ret
2258 SET_SIZE(cli)
2259
2260 #endif /* __lint */
2261
2262 #if defined(__lint)
2263
2264 dtrace_icookie_t
2265 dtrace_interrupt_disable(void)
2266 { return (0); }
2267
2268 #else /* __lint */
2269
2270 #if defined(__amd64)
2271
2272 ENTRY(dtrace_interrupt_disable)
2273 pushfq
2274 popq %rax
2275 #if defined(__xpv)
2276 leaq xpv_panicking, %rdi
2277 movl (%rdi), %edi
2278 cmpl $0, %edi
2279 jne .dtrace_interrupt_disable_done
2280 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2281 /*
2282 * Synthesize the PS_IE bit from the event mask bit
2283 */
2284 andq $_BITNOT(PS_IE), %rax
2285 testb $1, %dl
2286 jnz .dtrace_interrupt_disable_done
2287 orq $PS_IE, %rax
2288 #else
2289 CLI(%rdx)
2290 #endif
2291 .dtrace_interrupt_disable_done:
2292 ret
2293 SET_SIZE(dtrace_interrupt_disable)
2294
2295 #elif defined(__i386)
2296
2297 ENTRY(dtrace_interrupt_disable)
2298 pushfl
2299 popl %eax
2300 #if defined(__xpv)
2301 leal xpv_panicking, %edx
2302 movl (%edx), %edx
2303 cmpl $0, %edx
2304 jne .dtrace_interrupt_disable_done
2305 CLIRET(%edx, %cl) /* returns event mask in %cl */
2306 /*
2307 * Synthesize the PS_IE bit from the event mask bit
2308 */
2309 andl $_BITNOT(PS_IE), %eax
2310 testb $1, %cl
2311 jnz .dtrace_interrupt_disable_done
2312 orl $PS_IE, %eax
2313 #else
2314 CLI(%edx)
2315 #endif
2316 .dtrace_interrupt_disable_done:
2317 ret
2318 SET_SIZE(dtrace_interrupt_disable)
2319
2320 #endif /* __i386 */
2321 #endif /* __lint */
2322
2323 #if defined(__lint)
2324
2325 /*ARGSUSED*/
2326 void
2327 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2328 {}
2329
2330 #else /* __lint */
2331
2332 #if defined(__amd64)
2333
2334 ENTRY(dtrace_interrupt_enable)
2335 pushq %rdi
2336 popfq
2337 #if defined(__xpv)
2338 leaq xpv_panicking, %rdx
2339 movl (%rdx), %edx
2340 cmpl $0, %edx
2341 jne .dtrace_interrupt_enable_done
2342 /*
2343 * Since we're -really- running unprivileged, our attempt
2344 * to change the state of the IF bit will be ignored. The
2345 * virtual IF bit is tweaked by CLI and STI.
2346 */
2347 IE_TO_EVENT_MASK(%rdx, %rdi)
2348 #endif
2349 .dtrace_interrupt_enable_done:
2350 ret
2351 SET_SIZE(dtrace_interrupt_enable)
2352
2353 #elif defined(__i386)
2354
2355 ENTRY(dtrace_interrupt_enable)
2356 movl 4(%esp), %eax
2357 pushl %eax
2358 popfl
2359 #if defined(__xpv)
2360 leal xpv_panicking, %edx
2361 movl (%edx), %edx
2362 cmpl $0, %edx
2363 jne .dtrace_interrupt_enable_done
2364 /*
2365 * Since we're -really- running unprivileged, our attempt
2366 * to change the state of the IF bit will be ignored. The
2367 * virtual IF bit is tweaked by CLI and STI.
2368 */
2369 IE_TO_EVENT_MASK(%edx, %eax)
2370 #endif
2371 .dtrace_interrupt_enable_done:
2372 ret
2373 SET_SIZE(dtrace_interrupt_enable)
2374
2375 #endif /* __i386 */
2376 #endif /* __lint */
2377
2378
2379 #if defined(lint)
2380
2381 void
2382 dtrace_membar_producer(void)
2383 {}
2384
2385 void
2386 dtrace_membar_consumer(void)
2387 {}
2388
2389 #else /* __lint */
2390
2391 ENTRY(dtrace_membar_producer)
2392 rep; ret /* use 2 byte return instruction when branch target */
2393 /* AMD Software Optimization Guide - Section 6.2 */
2394 SET_SIZE(dtrace_membar_producer)
2395
2396 ENTRY(dtrace_membar_consumer)
2397 rep; ret /* use 2 byte return instruction when branch target */
2398 /* AMD Software Optimization Guide - Section 6.2 */
2399 SET_SIZE(dtrace_membar_consumer)
2400
2401 #endif /* __lint */
2402
2403 #if defined(__lint)
2404
2405 kthread_id_t
2406 threadp(void)
2407 { return ((kthread_id_t)0); }
2408
2409 #else /* __lint */
2410
2411 #if defined(__amd64)
2412
2413 ENTRY(threadp)
2414 movq %gs:CPU_THREAD, %rax
2415 ret
2416 SET_SIZE(threadp)
2417
2418 #elif defined(__i386)
2419
2420 ENTRY(threadp)
2421 movl %gs:CPU_THREAD, %eax
2422 ret
2423 SET_SIZE(threadp)
2424
2425 #endif /* __i386 */
2426 #endif /* __lint */
2427
2428 /*
2429 * Checksum routine for Internet Protocol Headers
2430 */
2431
2432 #if defined(__lint)
2433
2434 /* ARGSUSED */
2435 unsigned int
2436 ip_ocsum(
2437 ushort_t *address, /* ptr to 1st message buffer */
2438 int halfword_count, /* length of data */
2439 unsigned int sum) /* partial checksum */
2440 {
2441 int i;
2442 unsigned int psum = 0; /* partial sum */
2443
2444 for (i = 0; i < halfword_count; i++, address++) {
2445 psum += *address;
2446 }
2447
2448 while ((psum >> 16) != 0) {
2449 psum = (psum & 0xffff) + (psum >> 16);
2450 }
2451
2452 psum += sum;
2453
2454 while ((psum >> 16) != 0) {
2455 psum = (psum & 0xffff) + (psum >> 16);
2456 }
2457
2458 return (psum);
2459 }
2460
2461 #else /* __lint */
2462
2463 #if defined(__amd64)
2464
2465 ENTRY(ip_ocsum)
2466 pushq %rbp
2467 movq %rsp, %rbp
2468 #ifdef DEBUG
2469 movq postbootkernelbase(%rip), %rax
2470 cmpq %rax, %rdi
2471 jnb 1f
2472 xorl %eax, %eax
2473 movq %rdi, %rsi
2474 leaq .ip_ocsum_panic_msg(%rip), %rdi
2475 call panic
2476 /*NOTREACHED*/
2477 .ip_ocsum_panic_msg:
2478 .string "ip_ocsum: address 0x%p below kernelbase\n"
2479 1:
2480 #endif
2481 movl %esi, %ecx /* halfword_count */
2482 movq %rdi, %rsi /* address */
2483 /* partial sum in %edx */
2484 xorl %eax, %eax
2485 testl %ecx, %ecx
2486 jz .ip_ocsum_done
2487 testq $3, %rsi
2488 jnz .ip_csum_notaligned
2489 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2490 .next_iter:
2491 /* XX64 opportunities for prefetch? */
2492 /* XX64 compute csum with 64 bit quantities? */
2493 subl $32, %ecx
2494 jl .less_than_32
2495
2496 addl 0(%rsi), %edx
2497 .only60:
2498 adcl 4(%rsi), %eax
2499 .only56:
2500 adcl 8(%rsi), %edx
2501 .only52:
2502 adcl 12(%rsi), %eax
2503 .only48:
2504 adcl 16(%rsi), %edx
2505 .only44:
2506 adcl 20(%rsi), %eax
2507 .only40:
2508 adcl 24(%rsi), %edx
2509 .only36:
2510 adcl 28(%rsi), %eax
2511 .only32:
2512 adcl 32(%rsi), %edx
2513 .only28:
2514 adcl 36(%rsi), %eax
2515 .only24:
2516 adcl 40(%rsi), %edx
2517 .only20:
2518 adcl 44(%rsi), %eax
2519 .only16:
2520 adcl 48(%rsi), %edx
2521 .only12:
2522 adcl 52(%rsi), %eax
2523 .only8:
2524 adcl 56(%rsi), %edx
2525 .only4:
2526 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2527 .only0:
2528 adcl $0, %eax /* could be adding -1 in eax with a carry */
2529 adcl $0, %eax
2530
2531 addq $64, %rsi
2532 testl %ecx, %ecx
2533 jnz .next_iter
2534
2535 .ip_ocsum_done:
2536 addl %eax, %edx
2537 adcl $0, %edx
2538 movl %edx, %eax /* form a 16 bit checksum by */
2539 shrl $16, %eax /* adding two halves of 32 bit checksum */
2540 addw %dx, %ax
2541 adcw $0, %ax
2542 andl $0xffff, %eax
2543 leave
2544 ret
2545
2546 .ip_csum_notaligned:
2547 xorl %edi, %edi
2548 movw (%rsi), %di
2549 addl %edi, %edx
2550 adcl $0, %edx
2551 addq $2, %rsi
2552 decl %ecx
2553 jmp .ip_csum_aligned
2554
2555 .less_than_32:
2556 addl $32, %ecx
2557 testl $1, %ecx
2558 jz .size_aligned
2559 andl $0xfe, %ecx
2560 movzwl (%rsi, %rcx, 2), %edi
2561 addl %edi, %edx
2562 adcl $0, %edx
2563 .size_aligned:
2564 movl %ecx, %edi
2565 shrl $1, %ecx
2566 shl $1, %edi
2567 subq $64, %rdi
2568 addq %rdi, %rsi
2569 leaq .ip_ocsum_jmptbl(%rip), %rdi
2570 leaq (%rdi, %rcx, 8), %rdi
2571 xorl %ecx, %ecx
2572 clc
2573 jmp *(%rdi)
2574
2575 .align 8
2576 .ip_ocsum_jmptbl:
2577 .quad .only0, .only4, .only8, .only12, .only16, .only20
2578 .quad .only24, .only28, .only32, .only36, .only40, .only44
2579 .quad .only48, .only52, .only56, .only60
2580 SET_SIZE(ip_ocsum)
2581
2582 #elif defined(__i386)
2583
2584 ENTRY(ip_ocsum)
2585 pushl %ebp
2586 movl %esp, %ebp
2587 pushl %ebx
2588 pushl %esi
2589 pushl %edi
2590 movl 12(%ebp), %ecx /* count of half words */
2591 movl 16(%ebp), %edx /* partial checksum */
2592 movl 8(%ebp), %esi
2593 xorl %eax, %eax
2594 testl %ecx, %ecx
2595 jz .ip_ocsum_done
2596
2597 testl $3, %esi
2598 jnz .ip_csum_notaligned
2599 .ip_csum_aligned:
2600 .next_iter:
2601 subl $32, %ecx
2602 jl .less_than_32
2603
2604 addl 0(%esi), %edx
2605 .only60:
2606 adcl 4(%esi), %eax
2607 .only56:
2608 adcl 8(%esi), %edx
2609 .only52:
2610 adcl 12(%esi), %eax
2611 .only48:
2612 adcl 16(%esi), %edx
2613 .only44:
2614 adcl 20(%esi), %eax
2615 .only40:
2616 adcl 24(%esi), %edx
2617 .only36:
2618 adcl 28(%esi), %eax
2619 .only32:
2620 adcl 32(%esi), %edx
2621 .only28:
2622 adcl 36(%esi), %eax
2623 .only24:
2624 adcl 40(%esi), %edx
2625 .only20:
2626 adcl 44(%esi), %eax
2627 .only16:
2628 adcl 48(%esi), %edx
2629 .only12:
2630 adcl 52(%esi), %eax
2631 .only8:
2632 adcl 56(%esi), %edx
2633 .only4:
2634 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2635 .only0:
2636 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2637 adcl $0, %eax
2638
2639 addl $64, %esi
2640 andl %ecx, %ecx
2641 jnz .next_iter
2642
2643 .ip_ocsum_done:
2644 addl %eax, %edx
2645 adcl $0, %edx
2646 movl %edx, %eax /* form a 16 bit checksum by */
2647 shrl $16, %eax /* adding two halves of 32 bit checksum */
2648 addw %dx, %ax
2649 adcw $0, %ax
2650 andl $0xffff, %eax
2651 popl %edi /* restore registers */
2652 popl %esi
2653 popl %ebx
2654 leave
2655 ret
2656
2657 .ip_csum_notaligned:
2658 xorl %edi, %edi
2659 movw (%esi), %di
2660 addl %edi, %edx
2661 adcl $0, %edx
2662 addl $2, %esi
2663 decl %ecx
2664 jmp .ip_csum_aligned
2665
2666 .less_than_32:
2667 addl $32, %ecx
2668 testl $1, %ecx
2669 jz .size_aligned
2670 andl $0xfe, %ecx
2671 movzwl (%esi, %ecx, 2), %edi
2672 addl %edi, %edx
2673 adcl $0, %edx
2674 .size_aligned:
2675 movl %ecx, %edi
2676 shrl $1, %ecx
2677 shl $1, %edi
2678 subl $64, %edi
2679 addl %edi, %esi
2680 movl $.ip_ocsum_jmptbl, %edi
2681 lea (%edi, %ecx, 4), %edi
2682 xorl %ecx, %ecx
2683 clc
2684 jmp *(%edi)
2685 SET_SIZE(ip_ocsum)
2686
2687 .data
2688 .align 4
2689
2690 .ip_ocsum_jmptbl:
2691 .long .only0, .only4, .only8, .only12, .only16, .only20
2692 .long .only24, .only28, .only32, .only36, .only40, .only44
2693 .long .only48, .only52, .only56, .only60
2694
2695
2696 #endif /* __i386 */
2697 #endif /* __lint */
2698
2699 /*
2700 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2701 * Provided to manipulate hrtime_t values.
2702 */
2703 #if defined(__lint)
2704
2705 /* result = a * b; */
2706
2707 /* ARGSUSED */
2708 unsigned long long
2709 mul32(uint_t a, uint_t b)
2710 { return (0); }
2711
2712 #else /* __lint */
2713
2714 #if defined(__amd64)
2715
2716 ENTRY(mul32)
2717 xorl %edx, %edx /* XX64 joe, paranoia? */
2718 movl %edi, %eax
2719 mull %esi
2720 shlq $32, %rdx
2721 orq %rdx, %rax
2722 ret
2723 SET_SIZE(mul32)
2724
2725 #elif defined(__i386)
2726
2727 ENTRY(mul32)
2728 movl 8(%esp), %eax
2729 movl 4(%esp), %ecx
2730 mull %ecx
2731 ret
2732 SET_SIZE(mul32)
2733
2734 #endif /* __i386 */
2735 #endif /* __lint */
2736
2737 #if defined(notused)
2738 #if defined(__lint)
2739 /* ARGSUSED */
2740 void
2741 load_pte64(uint64_t *pte, uint64_t pte_value)
2742 {}
2743 #else /* __lint */
2744 .globl load_pte64
2745 load_pte64:
2746 movl 4(%esp), %eax
2747 movl 8(%esp), %ecx
2748 movl 12(%esp), %edx
2749 movl %edx, 4(%eax)
2750 movl %ecx, (%eax)
2751 ret
2752 #endif /* __lint */
2753 #endif /* notused */
2754
2755 #if defined(__lint)
2756
2757 /*ARGSUSED*/
2758 void
2759 scan_memory(caddr_t addr, size_t size)
2760 {}
2761
2762 #else /* __lint */
2763
2764 #if defined(__amd64)
2765
2766 ENTRY(scan_memory)
2767 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2768 jz .scanm_done
2769 movq %rsi, %rcx /* move count into rep control register */
2770 movq %rdi, %rsi /* move addr into lodsq control reg. */
2771 rep lodsq /* scan the memory range */
2772 .scanm_done:
2773 rep; ret /* use 2 byte return instruction when branch target */
2774 /* AMD Software Optimization Guide - Section 6.2 */
2775 SET_SIZE(scan_memory)
2776
2777 #elif defined(__i386)
2778
2779 ENTRY(scan_memory)
2780 pushl %ecx
2781 pushl %esi
2782 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2783 shrl $2, %ecx /* convert from byte count to word count */
2784 jz .scanm_done
2785 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2786 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2787 lodsl
2788 .scanm_done:
2789 popl %esi
2790 popl %ecx
2791 ret
2792 SET_SIZE(scan_memory)
2793
2794 #endif /* __i386 */
2795 #endif /* __lint */
2796
2797
2798 #if defined(__lint)
2799
2800 /*ARGSUSED */
2801 int
2802 lowbit(ulong_t i)
2803 { return (0); }
2804
2805 #else /* __lint */
2806
2807 #if defined(__amd64)
2808
2809 ENTRY(lowbit)
2810 movl $-1, %eax
2811 bsfq %rdi, %rdi
2812 cmovnz %edi, %eax
2813 incl %eax
2814 ret
2815 SET_SIZE(lowbit)
2816
2817 #elif defined(__i386)
2818
2819 ENTRY(lowbit)
2820 bsfl 4(%esp), %eax
2821 jz 0f
2822 incl %eax
2823 ret
2824 0:
2825 xorl %eax, %eax
2826 ret
2827 SET_SIZE(lowbit)
2828
2829 #endif /* __i386 */
2830 #endif /* __lint */
2831
2832 #if defined(__lint)
2833
2834 /*ARGSUSED*/
2835 int
2836 highbit(ulong_t i)
2837 { return (0); }
2838
2839 /*ARGSUSED*/
2840 int
2841 highbit64(uint64_t i)
2842 { return (0); }
2843
2844 #else /* __lint */
2845
2846 #if defined(__amd64)
2847
2848 ENTRY(highbit)
2849 ALTENTRY(highbit64)
2850 movl $-1, %eax
2851 bsrq %rdi, %rdi
2852 cmovnz %edi, %eax
2853 incl %eax
2854 ret
2855 SET_SIZE(highbit64)
2856 SET_SIZE(highbit)
2857
2858 #elif defined(__i386)
2859
2860 ENTRY(highbit)
2861 bsrl 4(%esp), %eax
2862 jz 0f
2863 incl %eax
2864 ret
2865 0:
2866 xorl %eax, %eax
2867 ret
2868 SET_SIZE(highbit)
2869
2870 ENTRY(highbit64)
2871 bsrl 8(%esp), %eax
2872 jz highbit
2873 addl $33, %eax
2874 ret
2875 SET_SIZE(highbit64)
2876
2877 #endif /* __i386 */
2878 #endif /* __lint */
2879
2880 #if defined(__lint)
2881
2882 /*ARGSUSED*/
2883 uint64_t
2884 rdmsr(uint_t r)
2885 { return (0); }
2886
2887 /*ARGSUSED*/
2888 void
2889 wrmsr(uint_t r, const uint64_t val)
2890 {}
2891
2892 /*ARGSUSED*/
2893 uint64_t
2894 xrdmsr(uint_t r)
2895 { return (0); }
2896
2897 /*ARGSUSED*/
2898 void
2899 xwrmsr(uint_t r, const uint64_t val)
2900 {}
2901
2902 void
2903 invalidate_cache(void)
2904 {}
2905
2906 /*ARGSUSED*/
2907 uint64_t
2908 get_xcr(uint_t r)
2909 { return (0); }
2910
2911 /*ARGSUSED*/
2912 void
2913 set_xcr(uint_t r, const uint64_t val)
2914 {}
2915
2916 #else /* __lint */
2917
2918 #define XMSR_ACCESS_VAL $0x9c5a203a
2919
2920 #if defined(__amd64)
2921
2922 ENTRY(rdmsr)
2923 movl %edi, %ecx
2924 rdmsr
2925 shlq $32, %rdx
2926 orq %rdx, %rax
2927 ret
2928 SET_SIZE(rdmsr)
2929
2930 ENTRY(wrmsr)
2931 movq %rsi, %rdx
2932 shrq $32, %rdx
2933 movl %esi, %eax
2934 movl %edi, %ecx
2935 wrmsr
2936 ret
2937 SET_SIZE(wrmsr)
2938
2939 ENTRY(xrdmsr)
2940 pushq %rbp
2941 movq %rsp, %rbp
2942 movl %edi, %ecx
2943 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2944 rdmsr
2945 shlq $32, %rdx
2946 orq %rdx, %rax
2947 leave
2948 ret
2949 SET_SIZE(xrdmsr)
2950
2951 ENTRY(xwrmsr)
2952 pushq %rbp
2953 movq %rsp, %rbp
2954 movl %edi, %ecx
2955 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2956 movq %rsi, %rdx
2957 shrq $32, %rdx
2958 movl %esi, %eax
2959 wrmsr
2960 leave
2961 ret
2962 SET_SIZE(xwrmsr)
2963
2964 ENTRY(get_xcr)
2965 movl %edi, %ecx
2966 #xgetbv
2967 .byte 0x0f,0x01,0xd0
2968 shlq $32, %rdx
2969 orq %rdx, %rax
2970 ret
2971 SET_SIZE(get_xcr)
2972
2973 ENTRY(set_xcr)
2974 movq %rsi, %rdx
2975 shrq $32, %rdx
2976 movl %esi, %eax
2977 movl %edi, %ecx
2978 #xsetbv
2979 .byte 0x0f,0x01,0xd1
2980 ret
2981 SET_SIZE(set_xcr)
2982
2983 #elif defined(__i386)
2984
2985 ENTRY(rdmsr)
2986 movl 4(%esp), %ecx
2987 rdmsr
2988 ret
2989 SET_SIZE(rdmsr)
2990
2991 ENTRY(wrmsr)
2992 movl 4(%esp), %ecx
2993 movl 8(%esp), %eax
2994 movl 12(%esp), %edx
2995 wrmsr
2996 ret
2997 SET_SIZE(wrmsr)
2998
2999 ENTRY(xrdmsr)
3000 pushl %ebp
3001 movl %esp, %ebp
3002 movl 8(%esp), %ecx
3003 pushl %edi
3004 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3005 rdmsr
3006 popl %edi
3007 leave
3008 ret
3009 SET_SIZE(xrdmsr)
3010
3011 ENTRY(xwrmsr)
3012 pushl %ebp
3013 movl %esp, %ebp
3014 movl 8(%esp), %ecx
3015 movl 12(%esp), %eax
3016 movl 16(%esp), %edx
3017 pushl %edi
3018 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3019 wrmsr
3020 popl %edi
3021 leave
3022 ret
3023 SET_SIZE(xwrmsr)
3024
3025 ENTRY(get_xcr)
3026 movl 4(%esp), %ecx
3027 #xgetbv
3028 .byte 0x0f,0x01,0xd0
3029 ret
3030 SET_SIZE(get_xcr)
3031
3032 ENTRY(set_xcr)
3033 movl 4(%esp), %ecx
3034 movl 8(%esp), %eax
3035 movl 12(%esp), %edx
3036 #xsetbv
3037 .byte 0x0f,0x01,0xd1
3038 ret
3039 SET_SIZE(set_xcr)
3040
3041 #endif /* __i386 */
3042
3043 ENTRY(invalidate_cache)
3044 wbinvd
3045 ret
3046 SET_SIZE(invalidate_cache)
3047
3048 #endif /* __lint */
3049
3050 #if defined(__lint)
3051
3052 /*ARGSUSED*/
3053 void
3054 getcregs(struct cregs *crp)
3055 {}
3056
3057 #else /* __lint */
3058
3059 #if defined(__amd64)
3060
3061 ENTRY_NP(getcregs)
3062 #if defined(__xpv)
3063 /*
3064 * Only a few of the hardware control registers or descriptor tables
3065 * are directly accessible to us, so just zero the structure.
3066 *
3067 * XXPV Perhaps it would be helpful for the hypervisor to return
3068 * virtualized versions of these for post-mortem use.
3069 * (Need to reevaluate - perhaps it already does!)
3070 */
3071 pushq %rdi /* save *crp */
3072 movq $CREGSZ, %rsi
3073 call bzero
3074 popq %rdi
3075
3076 /*
3077 * Dump what limited information we can
3078 */
3079 movq %cr0, %rax
3080 movq %rax, CREG_CR0(%rdi) /* cr0 */
3081 movq %cr2, %rax
3082 movq %rax, CREG_CR2(%rdi) /* cr2 */
3083 movq %cr3, %rax
3084 movq %rax, CREG_CR3(%rdi) /* cr3 */
3085 movq %cr4, %rax
3086 movq %rax, CREG_CR4(%rdi) /* cr4 */
3087
3088 #else /* __xpv */
3089
3090 #define GETMSR(r, off, d) \
3091 movl $r, %ecx; \
3092 rdmsr; \
3093 movl %eax, off(d); \
3094 movl %edx, off+4(d)
3095
3096 xorl %eax, %eax
3097 movq %rax, CREG_GDT+8(%rdi)
3098 sgdt CREG_GDT(%rdi) /* 10 bytes */
3099 movq %rax, CREG_IDT+8(%rdi)
3100 sidt CREG_IDT(%rdi) /* 10 bytes */
3101 movq %rax, CREG_LDT(%rdi)
3102 sldt CREG_LDT(%rdi) /* 2 bytes */
3103 movq %rax, CREG_TASKR(%rdi)
3104 str CREG_TASKR(%rdi) /* 2 bytes */
3105 movq %cr0, %rax
3106 movq %rax, CREG_CR0(%rdi) /* cr0 */
3107 movq %cr2, %rax
3108 movq %rax, CREG_CR2(%rdi) /* cr2 */
3109 movq %cr3, %rax
3110 movq %rax, CREG_CR3(%rdi) /* cr3 */
3111 movq %cr4, %rax
3112 movq %rax, CREG_CR4(%rdi) /* cr4 */
3113 movq %cr8, %rax
3114 movq %rax, CREG_CR8(%rdi) /* cr8 */
3115 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3116 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3117 #endif /* __xpv */
3118 ret
3119 SET_SIZE(getcregs)
3120
3121 #undef GETMSR
3122
3123 #elif defined(__i386)
3124
3125 ENTRY_NP(getcregs)
3126 #if defined(__xpv)
3127 /*
3128 * Only a few of the hardware control registers or descriptor tables
3129 * are directly accessible to us, so just zero the structure.
3130 *
3131 * XXPV Perhaps it would be helpful for the hypervisor to return
3132 * virtualized versions of these for post-mortem use.
3133 * (Need to reevaluate - perhaps it already does!)
3134 */
3135 movl 4(%esp), %edx
3136 pushl $CREGSZ
3137 pushl %edx
3138 call bzero
3139 addl $8, %esp
3140 movl 4(%esp), %edx
3141
3142 /*
3143 * Dump what limited information we can
3144 */
3145 movl %cr0, %eax
3146 movl %eax, CREG_CR0(%edx) /* cr0 */
3147 movl %cr2, %eax
3148 movl %eax, CREG_CR2(%edx) /* cr2 */
3149 movl %cr3, %eax
3150 movl %eax, CREG_CR3(%edx) /* cr3 */
3151 movl %cr4, %eax
3152 movl %eax, CREG_CR4(%edx) /* cr4 */
3153
3154 #else /* __xpv */
3155
3156 movl 4(%esp), %edx
3157 movw $0, CREG_GDT+6(%edx)
3158 movw $0, CREG_IDT+6(%edx)
3159 sgdt CREG_GDT(%edx) /* gdt */
3160 sidt CREG_IDT(%edx) /* idt */
3161 sldt CREG_LDT(%edx) /* ldt */
3162 str CREG_TASKR(%edx) /* task */
3163 movl %cr0, %eax
3164 movl %eax, CREG_CR0(%edx) /* cr0 */
3165 movl %cr2, %eax
3166 movl %eax, CREG_CR2(%edx) /* cr2 */
3167 movl %cr3, %eax
3168 movl %eax, CREG_CR3(%edx) /* cr3 */
3169 bt $X86FSET_LARGEPAGE, x86_featureset
3170 jnc .nocr4
3171 movl %cr4, %eax
3172 movl %eax, CREG_CR4(%edx) /* cr4 */
3173 jmp .skip
3174 .nocr4:
3175 movl $0, CREG_CR4(%edx)
3176 .skip:
3177 #endif
3178 ret
3179 SET_SIZE(getcregs)
3180
3181 #endif /* __i386 */
3182 #endif /* __lint */
3183
3184
3185 /*
3186 * A panic trigger is a word which is updated atomically and can only be set
3187 * once. We atomically store 0xDEFACEDD and load the old value. If the
3188 * previous value was 0, we succeed and return 1; otherwise return 0.
3189 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3190 * has its own version of this function to allow it to panic correctly from
3191 * probe context.
3192 */
3193 #if defined(__lint)
3194
3195 /*ARGSUSED*/
3196 int
3197 panic_trigger(int *tp)
3198 { return (0); }
3199
3200 /*ARGSUSED*/
3201 int
3202 dtrace_panic_trigger(int *tp)
3203 { return (0); }
3204
3205 #else /* __lint */
3206
3207 #if defined(__amd64)
3208
3209 ENTRY_NP(panic_trigger)
3210 xorl %eax, %eax
3211 movl $0xdefacedd, %edx
3212 lock
3213 xchgl %edx, (%rdi)
3214 cmpl $0, %edx
3215 je 0f
3216 movl $0, %eax
3217 ret
3218 0: movl $1, %eax
3219 ret
3220 SET_SIZE(panic_trigger)
3221
3222 ENTRY_NP(dtrace_panic_trigger)
3223 xorl %eax, %eax
3224 movl $0xdefacedd, %edx
3225 lock
3226 xchgl %edx, (%rdi)
3227 cmpl $0, %edx
3228 je 0f
3229 movl $0, %eax
3230 ret
3231 0: movl $1, %eax
3232 ret
3233 SET_SIZE(dtrace_panic_trigger)
3234
3235 #elif defined(__i386)
3236
3237 ENTRY_NP(panic_trigger)
3238 movl 4(%esp), %edx / %edx = address of trigger
3239 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3240 lock / assert lock
3241 xchgl %eax, (%edx) / exchange %eax and the trigger
3242 cmpl $0, %eax / if (%eax == 0x0)
3243 je 0f / return (1);
3244 movl $0, %eax / else
3245 ret / return (0);
3246 0: movl $1, %eax
3247 ret
3248 SET_SIZE(panic_trigger)
3249
3250 ENTRY_NP(dtrace_panic_trigger)
3251 movl 4(%esp), %edx / %edx = address of trigger
3252 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3253 lock / assert lock
3254 xchgl %eax, (%edx) / exchange %eax and the trigger
3255 cmpl $0, %eax / if (%eax == 0x0)
3256 je 0f / return (1);
3257 movl $0, %eax / else
3258 ret / return (0);
3259 0: movl $1, %eax
3260 ret
3261 SET_SIZE(dtrace_panic_trigger)
3262
3263 #endif /* __i386 */
3264 #endif /* __lint */
3265
3266 /*
3267 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3268 * into the panic code implemented in panicsys(). vpanic() is responsible
3269 * for passing through the format string and arguments, and constructing a
3270 * regs structure on the stack into which it saves the current register
3271 * values. If we are not dying due to a fatal trap, these registers will
3272 * then be preserved in panicbuf as the current processor state. Before
3273 * invoking panicsys(), vpanic() activates the first panic trigger (see
3274 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3275 * DTrace takes a slightly different panic path if it must panic from probe
3276 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3277 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3278 * branches back into vpanic().
3279 */
3280 #if defined(__lint)
3281
3282 /*ARGSUSED*/
3283 void
3284 vpanic(const char *format, va_list alist)
3285 {}
3286
3287 /*ARGSUSED*/
3288 void
3289 dtrace_vpanic(const char *format, va_list alist)
3290 {}
3291
3292 #else /* __lint */
3293
3294 #if defined(__amd64)
3295
3296 ENTRY_NP(vpanic) /* Initial stack layout: */
3297
3298 pushq %rbp /* | %rip | 0x60 */
3299 movq %rsp, %rbp /* | %rbp | 0x58 */
3300 pushfq /* | rfl | 0x50 */
3301 pushq %r11 /* | %r11 | 0x48 */
3302 pushq %r10 /* | %r10 | 0x40 */
3303 pushq %rbx /* | %rbx | 0x38 */
3304 pushq %rax /* | %rax | 0x30 */
3305 pushq %r9 /* | %r9 | 0x28 */
3306 pushq %r8 /* | %r8 | 0x20 */
3307 pushq %rcx /* | %rcx | 0x18 */
3308 pushq %rdx /* | %rdx | 0x10 */
3309 pushq %rsi /* | %rsi | 0x8 alist */
3310 pushq %rdi /* | %rdi | 0x0 format */
3311
3312 movq %rsp, %rbx /* %rbx = current %rsp */
3313
3314 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3315 call panic_trigger /* %eax = panic_trigger() */
3316
3317 vpanic_common:
3318 /*
3319 * The panic_trigger result is in %eax from the call above, and
3320 * dtrace_panic places it in %eax before branching here.
3321 * The rdmsr instructions that follow below will clobber %eax so
3322 * we stash the panic_trigger result in %r11d.
3323 */
3324 movl %eax, %r11d
3325 cmpl $0, %r11d
3326 je 0f
3327
3328 /*
3329 * If panic_trigger() was successful, we are the first to initiate a
3330 * panic: we now switch to the reserved panic_stack before continuing.
3331 */
3332 leaq panic_stack(%rip), %rsp
3333 addq $PANICSTKSIZE, %rsp
3334 0: subq $REGSIZE, %rsp
3335 /*
3336 * Now that we've got everything set up, store the register values as
3337 * they were when we entered vpanic() to the designated location in
3338 * the regs structure we allocated on the stack.
3339 */
3340 movq 0x0(%rbx), %rcx
3341 movq %rcx, REGOFF_RDI(%rsp)
3342 movq 0x8(%rbx), %rcx
3343 movq %rcx, REGOFF_RSI(%rsp)
3344 movq 0x10(%rbx), %rcx
3345 movq %rcx, REGOFF_RDX(%rsp)
3346 movq 0x18(%rbx), %rcx
3347 movq %rcx, REGOFF_RCX(%rsp)
3348 movq 0x20(%rbx), %rcx
3349
3350 movq %rcx, REGOFF_R8(%rsp)
3351 movq 0x28(%rbx), %rcx
3352 movq %rcx, REGOFF_R9(%rsp)
3353 movq 0x30(%rbx), %rcx
3354 movq %rcx, REGOFF_RAX(%rsp)
3355 movq 0x38(%rbx), %rcx
3356 movq %rcx, REGOFF_RBX(%rsp)
3357 movq 0x58(%rbx), %rcx
3358
3359 movq %rcx, REGOFF_RBP(%rsp)
3360 movq 0x40(%rbx), %rcx
3361 movq %rcx, REGOFF_R10(%rsp)
3362 movq 0x48(%rbx), %rcx
3363 movq %rcx, REGOFF_R11(%rsp)
3364 movq %r12, REGOFF_R12(%rsp)
3365
3366 movq %r13, REGOFF_R13(%rsp)
3367 movq %r14, REGOFF_R14(%rsp)
3368 movq %r15, REGOFF_R15(%rsp)
3369
3370 xorl %ecx, %ecx
3371 movw %ds, %cx
3372 movq %rcx, REGOFF_DS(%rsp)
3373 movw %es, %cx
3374 movq %rcx, REGOFF_ES(%rsp)
3375 movw %fs, %cx
3376 movq %rcx, REGOFF_FS(%rsp)
3377 movw %gs, %cx
3378 movq %rcx, REGOFF_GS(%rsp)
3379
3380 movq $0, REGOFF_TRAPNO(%rsp)
3381
3382 movq $0, REGOFF_ERR(%rsp)
3383 leaq vpanic(%rip), %rcx
3384 movq %rcx, REGOFF_RIP(%rsp)
3385 movw %cs, %cx
3386 movzwq %cx, %rcx
3387 movq %rcx, REGOFF_CS(%rsp)
3388 movq 0x50(%rbx), %rcx
3389 movq %rcx, REGOFF_RFL(%rsp)
3390 movq %rbx, %rcx
3391 addq $0x60, %rcx
3392 movq %rcx, REGOFF_RSP(%rsp)
3393 movw %ss, %cx
3394 movzwq %cx, %rcx
3395 movq %rcx, REGOFF_SS(%rsp)
3396
3397 /*
3398 * panicsys(format, alist, rp, on_panic_stack)
3399 */
3400 movq REGOFF_RDI(%rsp), %rdi /* format */
3401 movq REGOFF_RSI(%rsp), %rsi /* alist */
3402 movq %rsp, %rdx /* struct regs */
3403 movl %r11d, %ecx /* on_panic_stack */
3404 call panicsys
3405 addq $REGSIZE, %rsp
3406 popq %rdi
3407 popq %rsi
3408 popq %rdx
3409 popq %rcx
3410 popq %r8
3411 popq %r9
3412 popq %rax
3413 popq %rbx
3414 popq %r10
3415 popq %r11
3416 popfq
3417 leave
3418 ret
3419 SET_SIZE(vpanic)
3420
3421 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3422
3423 pushq %rbp /* | %rip | 0x60 */
3424 movq %rsp, %rbp /* | %rbp | 0x58 */
3425 pushfq /* | rfl | 0x50 */
3426 pushq %r11 /* | %r11 | 0x48 */
3427 pushq %r10 /* | %r10 | 0x40 */
3428 pushq %rbx /* | %rbx | 0x38 */
3429 pushq %rax /* | %rax | 0x30 */
3430 pushq %r9 /* | %r9 | 0x28 */
3431 pushq %r8 /* | %r8 | 0x20 */
3432 pushq %rcx /* | %rcx | 0x18 */
3433 pushq %rdx /* | %rdx | 0x10 */
3434 pushq %rsi /* | %rsi | 0x8 alist */
3435 pushq %rdi /* | %rdi | 0x0 format */
3436
3437 movq %rsp, %rbx /* %rbx = current %rsp */
3438
3439 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3440 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3441 jmp vpanic_common
3442
3443 SET_SIZE(dtrace_vpanic)
3444
3445 #elif defined(__i386)
3446
3447 ENTRY_NP(vpanic) / Initial stack layout:
3448
3449 pushl %ebp / | %eip | 20
3450 movl %esp, %ebp / | %ebp | 16
3451 pushl %eax / | %eax | 12
3452 pushl %ebx / | %ebx | 8
3453 pushl %ecx / | %ecx | 4
3454 pushl %edx / | %edx | 0
3455
3456 movl %esp, %ebx / %ebx = current stack pointer
3457
3458 lea panic_quiesce, %eax / %eax = &panic_quiesce
3459 pushl %eax / push &panic_quiesce
3460 call panic_trigger / %eax = panic_trigger()
3461 addl $4, %esp / reset stack pointer
3462
3463 vpanic_common:
3464 cmpl $0, %eax / if (%eax == 0)
3465 je 0f / goto 0f;
3466
3467 /*
3468 * If panic_trigger() was successful, we are the first to initiate a
3469 * panic: we now switch to the reserved panic_stack before continuing.
3470 */
3471 lea panic_stack, %esp / %esp = panic_stack
3472 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3473
3474 0: subl $REGSIZE, %esp / allocate struct regs
3475
3476 /*
3477 * Now that we've got everything set up, store the register values as
3478 * they were when we entered vpanic() to the designated location in
3479 * the regs structure we allocated on the stack.
3480 */
3481 #if !defined(__GNUC_AS__)
3482 movw %gs, %edx
3483 movl %edx, REGOFF_GS(%esp)
3484 movw %fs, %edx
3485 movl %edx, REGOFF_FS(%esp)
3486 movw %es, %edx
3487 movl %edx, REGOFF_ES(%esp)
3488 movw %ds, %edx
3489 movl %edx, REGOFF_DS(%esp)
3490 #else /* __GNUC_AS__ */
3491 mov %gs, %edx
3492 mov %edx, REGOFF_GS(%esp)
3493 mov %fs, %edx
3494 mov %edx, REGOFF_FS(%esp)
3495 mov %es, %edx
3496 mov %edx, REGOFF_ES(%esp)
3497 mov %ds, %edx
3498 mov %edx, REGOFF_DS(%esp)
3499 #endif /* __GNUC_AS__ */
3500 movl %edi, REGOFF_EDI(%esp)
3501 movl %esi, REGOFF_ESI(%esp)
3502 movl 16(%ebx), %ecx
3503 movl %ecx, REGOFF_EBP(%esp)
3504 movl %ebx, %ecx
3505 addl $20, %ecx
3506 movl %ecx, REGOFF_ESP(%esp)
3507 movl 8(%ebx), %ecx
3508 movl %ecx, REGOFF_EBX(%esp)
3509 movl 0(%ebx), %ecx
3510 movl %ecx, REGOFF_EDX(%esp)
3511 movl 4(%ebx), %ecx
3512 movl %ecx, REGOFF_ECX(%esp)
3513 movl 12(%ebx), %ecx
3514 movl %ecx, REGOFF_EAX(%esp)
3515 movl $0, REGOFF_TRAPNO(%esp)
3516 movl $0, REGOFF_ERR(%esp)
3517 lea vpanic, %ecx
3518 movl %ecx, REGOFF_EIP(%esp)
3519 #if !defined(__GNUC_AS__)
3520 movw %cs, %edx
3521 #else /* __GNUC_AS__ */
3522 mov %cs, %edx
3523 #endif /* __GNUC_AS__ */
3524 movl %edx, REGOFF_CS(%esp)
3525 pushfl
3526 popl %ecx
3527 #if defined(__xpv)
3528 /*
3529 * Synthesize the PS_IE bit from the event mask bit
3530 */
3531 CURTHREAD(%edx)
3532 KPREEMPT_DISABLE(%edx)
3533 EVENT_MASK_TO_IE(%edx, %ecx)
3534 CURTHREAD(%edx)
3535 KPREEMPT_ENABLE_NOKP(%edx)
3536 #endif
3537 movl %ecx, REGOFF_EFL(%esp)
3538 movl $0, REGOFF_UESP(%esp)
3539 #if !defined(__GNUC_AS__)
3540 movw %ss, %edx
3541 #else /* __GNUC_AS__ */
3542 mov %ss, %edx
3543 #endif /* __GNUC_AS__ */
3544 movl %edx, REGOFF_SS(%esp)
3545
3546 movl %esp, %ecx / %ecx = ®s
3547 pushl %eax / push on_panic_stack
3548 pushl %ecx / push ®s
3549 movl 12(%ebp), %ecx / %ecx = alist
3550 pushl %ecx / push alist
3551 movl 8(%ebp), %ecx / %ecx = format
3552 pushl %ecx / push format
3553 call panicsys / panicsys();
3554 addl $16, %esp / pop arguments
3555
3556 addl $REGSIZE, %esp
3557 popl %edx
3558 popl %ecx
3559 popl %ebx
3560 popl %eax
3561 leave
3562 ret
3563 SET_SIZE(vpanic)
3564
3565 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3566
3567 pushl %ebp / | %eip | 20
3568 movl %esp, %ebp / | %ebp | 16
3569 pushl %eax / | %eax | 12
3570 pushl %ebx / | %ebx | 8
3571 pushl %ecx / | %ecx | 4
3572 pushl %edx / | %edx | 0
3573
3574 movl %esp, %ebx / %ebx = current stack pointer
3575
3576 lea panic_quiesce, %eax / %eax = &panic_quiesce
3577 pushl %eax / push &panic_quiesce
3578 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3579 addl $4, %esp / reset stack pointer
3580 jmp vpanic_common / jump back to common code
3581
3582 SET_SIZE(dtrace_vpanic)
3583
3584 #endif /* __i386 */
3585 #endif /* __lint */
3586
3587 #if defined(__lint)
3588
3589 void
3590 hres_tick(void)
3591 {}
3592
3593 int64_t timedelta;
3594 hrtime_t hrtime_base;
3595
3596 #else /* __lint */
3597
3598 DGDEF3(timedelta, 8, 8)
3599 .long 0, 0
3600
3601 /*
3602 * initialized to a non zero value to make pc_gethrtime()
3603 * work correctly even before clock is initialized
3604 */
3605 DGDEF3(hrtime_base, 8, 8)
3606 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3607
3608 DGDEF3(adj_shift, 4, 4)
3609 .long ADJ_SHIFT
3610
3611 #if defined(__amd64)
3612
3613 ENTRY_NP(hres_tick)
3614 pushq %rbp
3615 movq %rsp, %rbp
3616
3617 /*
3618 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3619 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3620 * At worst, performing this now instead of under CLOCK_LOCK may
3621 * introduce some jitter in pc_gethrestime().
3622 */
3623 call *gethrtimef(%rip)
3624 movq %rax, %r8
3625
3626 leaq hres_lock(%rip), %rax
3627 movb $-1, %dl
3628 .CL1:
3629 xchgb %dl, (%rax)
3630 testb %dl, %dl
3631 jz .CL3 /* got it */
3632 .CL2:
3633 cmpb $0, (%rax) /* possible to get lock? */
3634 pause
3635 jne .CL2
3636 jmp .CL1 /* yes, try again */
3637 .CL3:
3638 /*
3639 * compute the interval since last time hres_tick was called
3640 * and adjust hrtime_base and hrestime accordingly
3641 * hrtime_base is an 8 byte value (in nsec), hrestime is
3642 * a timestruc_t (sec, nsec)
3643 */
3644 leaq hres_last_tick(%rip), %rax
3645 movq %r8, %r11
3646 subq (%rax), %r8
3647 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3648 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3649 /*
3650 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3651 */
3652 movq %r11, (%rax)
3653
3654 call __adj_hrestime
3655
3656 /*
3657 * release the hres_lock
3658 */
3659 incl hres_lock(%rip)
3660 leave
3661 ret
3662 SET_SIZE(hres_tick)
3663
3664 #elif defined(__i386)
3665
3666 ENTRY_NP(hres_tick)
3667 pushl %ebp
3668 movl %esp, %ebp
3669 pushl %esi
3670 pushl %ebx
3671
3672 /*
3673 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3674 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3675 * At worst, performing this now instead of under CLOCK_LOCK may
3676 * introduce some jitter in pc_gethrestime().
3677 */
3678 call *gethrtimef
3679 movl %eax, %ebx
3680 movl %edx, %esi
3681
3682 movl $hres_lock, %eax
3683 movl $-1, %edx
3684 .CL1:
3685 xchgb %dl, (%eax)
3686 testb %dl, %dl
3687 jz .CL3 / got it
3688 .CL2:
3689 cmpb $0, (%eax) / possible to get lock?
3690 pause
3691 jne .CL2
3692 jmp .CL1 / yes, try again
3693 .CL3:
3694 /*
3695 * compute the interval since last time hres_tick was called
3696 * and adjust hrtime_base and hrestime accordingly
3697 * hrtime_base is an 8 byte value (in nsec), hrestime is
3698 * timestruc_t (sec, nsec)
3699 */
3700
3701 lea hres_last_tick, %eax
3702
3703 movl %ebx, %edx
3704 movl %esi, %ecx
3705
3706 subl (%eax), %edx
3707 sbbl 4(%eax), %ecx
3708
3709 addl %edx, hrtime_base / add interval to hrtime_base
3710 adcl %ecx, hrtime_base+4
3711
3712 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3713
3714 /
3715 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3716 /
3717 movl %ebx, (%eax)
3718 movl %esi, 4(%eax)
3719
3720 / get hrestime at this moment. used as base for pc_gethrestime
3721 /
3722 / Apply adjustment, if any
3723 /
3724 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3725 / (max_hres_adj)
3726 /
3727 / void
3728 / adj_hrestime()
3729 / {
3730 / long long adj;
3731 /
3732 / if (hrestime_adj == 0)
3733 / adj = 0;
3734 / else if (hrestime_adj > 0) {
3735 / if (hrestime_adj < HRES_ADJ)
3736 / adj = hrestime_adj;
3737 / else
3738 / adj = HRES_ADJ;
3739 / }
3740 / else {
3741 / if (hrestime_adj < -(HRES_ADJ))
3742 / adj = -(HRES_ADJ);
3743 / else
3744 / adj = hrestime_adj;
3745 / }
3746 /
3747 / timedelta -= adj;
3748 / hrestime_adj = timedelta;
3749 / hrestime.tv_nsec += adj;
3750 /
3751 / while (hrestime.tv_nsec >= NANOSEC) {
3752 / one_sec++;
3753 / hrestime.tv_sec++;
3754 / hrestime.tv_nsec -= NANOSEC;
3755 / }
3756 / }
3757 __adj_hrestime:
3758 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3759 movl hrestime_adj+4, %edx
3760 andl %esi, %esi
3761 jne .CL4 / no
3762 andl %edx, %edx
3763 jne .CL4 / no
3764 subl %ecx, %ecx / yes, adj = 0;
3765 subl %edx, %edx
3766 jmp .CL5
3767 .CL4:
3768 subl %ecx, %ecx
3769 subl %eax, %eax
3770 subl %esi, %ecx
3771 sbbl %edx, %eax
3772 andl %eax, %eax / if (hrestime_adj > 0)
3773 jge .CL6
3774
3775 / In the following comments, HRES_ADJ is used, while in the code
3776 / max_hres_adj is used.
3777 /
3778 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3779 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3780 / on the logical equivalence of:
3781 /
3782 / !(hrestime_adj < HRES_ADJ)
3783 /
3784 / and the two step sequence:
3785 /
3786 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3787 /
3788 / which computes whether or not the least significant 32-bits
3789 / of hrestime_adj is greater than HRES_ADJ, followed by:
3790 /
3791 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3792 /
3793 / which generates a carry whenever step 1 is true or the most
3794 / significant long of the longlong hrestime_adj is non-zero.
3795
3796 movl max_hres_adj, %ecx / hrestime_adj is positive
3797 subl %esi, %ecx
3798 movl %edx, %eax
3799 adcl $-1, %eax
3800 jnc .CL7
3801 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3802 subl %edx, %edx
3803 jmp .CL5
3804
3805 / The following computation is similar to the one above.
3806 /
3807 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3808 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3809 / on the logical equivalence of:
3810 /
3811 / (hrestime_adj > -HRES_ADJ)
3812 /
3813 / and the two step sequence:
3814 /
3815 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3816 /
3817 / which means the least significant 32-bits of hrestime_adj is
3818 / greater than -HRES_ADJ, followed by:
3819 /
3820 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3821 /
3822 / which generates a carry only when step 1 is true and the most
3823 / significant long of the longlong hrestime_adj is -1.
3824
3825 .CL6: / hrestime_adj is negative
3826 movl %esi, %ecx
3827 addl max_hres_adj, %ecx
3828 movl %edx, %eax
3829 adcl $0, %eax
3830 jc .CL7
3831 xor %ecx, %ecx
3832 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3833 movl $-1, %edx
3834 jmp .CL5
3835 .CL7:
3836 movl %esi, %ecx / adj = hrestime_adj;
3837 .CL5:
3838 movl timedelta, %esi
3839 subl %ecx, %esi
3840 movl timedelta+4, %eax
3841 sbbl %edx, %eax
3842 movl %esi, timedelta
3843 movl %eax, timedelta+4 / timedelta -= adj;
3844 movl %esi, hrestime_adj
3845 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3846 addl hrestime+4, %ecx
3847
3848 movl %ecx, %eax / eax = tv_nsec
3849 1:
3850 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3851 jb .CL8 / no
3852 incl one_sec / yes, one_sec++;
3853 incl hrestime / hrestime.tv_sec++;
3854 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3855 jmp 1b / check for more seconds
3856
3857 .CL8:
3858 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3859 incl hres_lock / release the hres_lock
3860
3861 popl %ebx
3862 popl %esi
3863 leave
3864 ret
3865 SET_SIZE(hres_tick)
3866
3867 #endif /* __i386 */
3868 #endif /* __lint */
3869
3870 /*
3871 * void prefetch_smap_w(void *)
3872 *
3873 * Prefetch ahead within a linear list of smap structures.
3874 * Not implemented for ia32. Stub for compatibility.
3875 */
3876
3877 #if defined(__lint)
3878
3879 /*ARGSUSED*/
3880 void prefetch_smap_w(void *smp)
3881 {}
3882
3883 #else /* __lint */
3884
3885 ENTRY(prefetch_smap_w)
3886 rep; ret /* use 2 byte return instruction when branch target */
3887 /* AMD Software Optimization Guide - Section 6.2 */
3888 SET_SIZE(prefetch_smap_w)
3889
3890 #endif /* __lint */
3891
3892 /*
3893 * prefetch_page_r(page_t *)
3894 * issue prefetch instructions for a page_t
3895 */
3896 #if defined(__lint)
3897
3898 /*ARGSUSED*/
3899 void
3900 prefetch_page_r(void *pp)
3901 {}
3902
3903 #else /* __lint */
3904
3905 ENTRY(prefetch_page_r)
3906 rep; ret /* use 2 byte return instruction when branch target */
3907 /* AMD Software Optimization Guide - Section 6.2 */
3908 SET_SIZE(prefetch_page_r)
3909
3910 #endif /* __lint */
3911
3912 #if defined(__lint)
3913
3914 /*ARGSUSED*/
3915 int
3916 bcmp(const void *s1, const void *s2, size_t count)
3917 { return (0); }
3918
3919 #else /* __lint */
3920
3921 #if defined(__amd64)
3922
3923 ENTRY(bcmp)
3924 pushq %rbp
3925 movq %rsp, %rbp
3926 #ifdef DEBUG
3927 testq %rdx,%rdx
3928 je 1f
3929 movq postbootkernelbase(%rip), %r11
3930 cmpq %r11, %rdi
3931 jb 0f
3932 cmpq %r11, %rsi
3933 jnb 1f
3934 0: leaq .bcmp_panic_msg(%rip), %rdi
3935 xorl %eax, %eax
3936 call panic
3937 1:
3938 #endif /* DEBUG */
3939 call memcmp
3940 testl %eax, %eax
3941 setne %dl
3942 leave
3943 movzbl %dl, %eax
3944 ret
3945 SET_SIZE(bcmp)
3946
3947 #elif defined(__i386)
3948
3949 #define ARG_S1 8
3950 #define ARG_S2 12
3951 #define ARG_LENGTH 16
3952
3953 ENTRY(bcmp)
3954 pushl %ebp
3955 movl %esp, %ebp / create new stack frame
3956 #ifdef DEBUG
3957 cmpl $0, ARG_LENGTH(%ebp)
3958 je 1f
3959 movl postbootkernelbase, %eax
3960 cmpl %eax, ARG_S1(%ebp)
3961 jb 0f
3962 cmpl %eax, ARG_S2(%ebp)
3963 jnb 1f
3964 0: pushl $.bcmp_panic_msg
3965 call panic
3966 1:
3967 #endif /* DEBUG */
3968
3969 pushl %edi / save register variable
3970 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3971 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3972 cmpl %eax, %ecx / if the same string
3973 je .equal / goto .equal
3974 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3975 cmpl $4, %edi / if %edi < 4
3976 jb .byte_check / goto .byte_check
3977 .align 4
3978 .word_loop:
3979 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3980 leal -4(%edi), %edi / %edi -= 4
3981 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3982 jne .word_not_equal / if not equal, goto .word_not_equal
3983 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3984 leal 4(%eax), %eax / %eax += 4 (next word)
3985 cmpl $4, %edi / if %edi >= 4
3986 jae .word_loop / goto .word_loop
3987 .byte_check:
3988 cmpl $0, %edi / if %edi == 0
3989 je .equal / goto .equal
3990 jmp .byte_loop / goto .byte_loop (checks in bytes)
3991 .word_not_equal:
3992 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3993 .align 4
3994 .byte_loop:
3995 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3996 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3997 jne .not_equal / if not equal, goto .not_equal
3998 incl %ecx / %ecx++ (next byte)
3999 incl %eax / %eax++ (next byte)
4000 decl %edi / %edi--
4001 jnz .byte_loop / if not zero, goto .byte_loop
4002 .equal:
4003 xorl %eax, %eax / %eax = 0
4004 popl %edi / restore register variable
4005 leave / restore old stack frame
4006 ret / return (NULL)
4007 .align 4
4008 .not_equal:
4009 movl $1, %eax / return 1
4010 popl %edi / restore register variable
4011 leave / restore old stack frame
4012 ret / return (NULL)
4013 SET_SIZE(bcmp)
4014
4015 #endif /* __i386 */
4016
4017 #ifdef DEBUG
4018 .text
4019 .bcmp_panic_msg:
4020 .string "bcmp: arguments below kernelbase"
4021 #endif /* DEBUG */
4022
4023 #endif /* __lint */
4024
4025 #if defined(__lint)
4026
4027 uint_t
4028 bsrw_insn(uint16_t mask)
4029 {
4030 uint_t index = sizeof (mask) * NBBY - 1;
4031
4032 while ((mask & (1 << index)) == 0)
4033 index--;
4034 return (index);
4035 }
4036
4037 #else /* __lint */
4038
4039 #if defined(__amd64)
4040
4041 ENTRY_NP(bsrw_insn)
4042 xorl %eax, %eax
4043 bsrw %di, %ax
4044 ret
4045 SET_SIZE(bsrw_insn)
4046
4047 #elif defined(__i386)
4048
4049 ENTRY_NP(bsrw_insn)
4050 movw 4(%esp), %cx
4051 xorl %eax, %eax
4052 bsrw %cx, %ax
4053 ret
4054 SET_SIZE(bsrw_insn)
4055
4056 #endif /* __i386 */
4057 #endif /* __lint */
4058
4059 #if defined(__lint)
4060
4061 uint_t
4062 atomic_btr32(uint32_t *pending, uint_t pil)
4063 {
4064 return (*pending &= ~(1 << pil));
4065 }
4066
4067 #else /* __lint */
4068
4069 #if defined(__i386)
4070
4071 ENTRY_NP(atomic_btr32)
4072 movl 4(%esp), %ecx
4073 movl 8(%esp), %edx
4074 xorl %eax, %eax
4075 lock
4076 btrl %edx, (%ecx)
4077 setc %al
4078 ret
4079 SET_SIZE(atomic_btr32)
4080
4081 #endif /* __i386 */
4082 #endif /* __lint */
4083
4084 #if defined(__lint)
4085
4086 /*ARGSUSED*/
4087 void
4088 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4089 uint_t arg2)
4090 {}
4091
4092 #else /* __lint */
4093
4094 #if defined(__amd64)
4095
4096 ENTRY_NP(switch_sp_and_call)
4097 pushq %rbp
4098 movq %rsp, %rbp /* set up stack frame */
4099 movq %rdi, %rsp /* switch stack pointer */
4100 movq %rdx, %rdi /* pass func arg 1 */
4101 movq %rsi, %r11 /* save function to call */
4102 movq %rcx, %rsi /* pass func arg 2 */
4103 call *%r11 /* call function */
4104 leave /* restore stack */
4105 ret
4106 SET_SIZE(switch_sp_and_call)
4107
4108 #elif defined(__i386)
4109
4110 ENTRY_NP(switch_sp_and_call)
4111 pushl %ebp
4112 mov %esp, %ebp /* set up stack frame */
4113 movl 8(%ebp), %esp /* switch stack pointer */
4114 pushl 20(%ebp) /* push func arg 2 */
4115 pushl 16(%ebp) /* push func arg 1 */
4116 call *12(%ebp) /* call function */
4117 addl $8, %esp /* pop arguments */
4118 leave /* restore stack */
4119 ret
4120 SET_SIZE(switch_sp_and_call)
4121
4122 #endif /* __i386 */
4123 #endif /* __lint */
4124
4125 #if defined(__lint)
4126
4127 void
4128 kmdb_enter(void)
4129 {}
4130
4131 #else /* __lint */
4132
4133 #if defined(__amd64)
4134
4135 ENTRY_NP(kmdb_enter)
4136 pushq %rbp
4137 movq %rsp, %rbp
4138
4139 /*
4140 * Save flags, do a 'cli' then return the saved flags
4141 */
4142 call intr_clear
4143
4144 int $T_DBGENTR
4145
4146 /*
4147 * Restore the saved flags
4148 */
4149 movq %rax, %rdi
4150 call intr_restore
4151
4152 leave
4153 ret
4154 SET_SIZE(kmdb_enter)
4155
4156 #elif defined(__i386)
4157
4158 ENTRY_NP(kmdb_enter)
4159 pushl %ebp
4160 movl %esp, %ebp
4161
4162 /*
4163 * Save flags, do a 'cli' then return the saved flags
4164 */
4165 call intr_clear
4166
4167 int $T_DBGENTR
4168
4169 /*
4170 * Restore the saved flags
4171 */
4172 pushl %eax
4173 call intr_restore
4174 addl $4, %esp
4175
4176 leave
4177 ret
4178 SET_SIZE(kmdb_enter)
4179
4180 #endif /* __i386 */
4181 #endif /* __lint */
4182
4183 #if defined(__lint)
4184
4185 void
4186 return_instr(void)
4187 {}
4188
4189 #else /* __lint */
4190
4191 ENTRY_NP(return_instr)
4192 rep; ret /* use 2 byte instruction when branch target */
4193 /* AMD Software Optimization Guide - Section 6.2 */
4194 SET_SIZE(return_instr)
4195
4196 #endif /* __lint */
4197
4198 #if defined(__lint)
4199
4200 ulong_t
4201 getflags(void)
4202 {
4203 return (0);
4204 }
4205
4206 #else /* __lint */
4207
4208 #if defined(__amd64)
4209
4210 ENTRY(getflags)
4211 pushfq
4212 popq %rax
4213 #if defined(__xpv)
4214 CURTHREAD(%rdi)
4215 KPREEMPT_DISABLE(%rdi)
4216 /*
4217 * Synthesize the PS_IE bit from the event mask bit
4218 */
4219 CURVCPU(%r11)
4220 andq $_BITNOT(PS_IE), %rax
4221 XEN_TEST_UPCALL_MASK(%r11)
4222 jnz 1f
4223 orq $PS_IE, %rax
4224 1:
4225 KPREEMPT_ENABLE_NOKP(%rdi)
4226 #endif
4227 ret
4228 SET_SIZE(getflags)
4229
4230 #elif defined(__i386)
4231
4232 ENTRY(getflags)
4233 pushfl
4234 popl %eax
4235 #if defined(__xpv)
4236 CURTHREAD(%ecx)
4237 KPREEMPT_DISABLE(%ecx)
4238 /*
4239 * Synthesize the PS_IE bit from the event mask bit
4240 */
4241 CURVCPU(%edx)
4242 andl $_BITNOT(PS_IE), %eax
4243 XEN_TEST_UPCALL_MASK(%edx)
4244 jnz 1f
4245 orl $PS_IE, %eax
4246 1:
4247 KPREEMPT_ENABLE_NOKP(%ecx)
4248 #endif
4249 ret
4250 SET_SIZE(getflags)
4251
4252 #endif /* __i386 */
4253
4254 #endif /* __lint */
4255
4256 #if defined(__lint)
4257
4258 ftrace_icookie_t
4259 ftrace_interrupt_disable(void)
4260 { return (0); }
4261
4262 #else /* __lint */
4263
4264 #if defined(__amd64)
4265
4266 ENTRY(ftrace_interrupt_disable)
4267 pushfq
4268 popq %rax
4269 CLI(%rdx)
4270 ret
4271 SET_SIZE(ftrace_interrupt_disable)
4272
4273 #elif defined(__i386)
4274
4275 ENTRY(ftrace_interrupt_disable)
4276 pushfl
4277 popl %eax
4278 CLI(%edx)
4279 ret
4280 SET_SIZE(ftrace_interrupt_disable)
4281
4282 #endif /* __i386 */
4283 #endif /* __lint */
4284
4285 #if defined(__lint)
4286
4287 /*ARGSUSED*/
4288 void
4289 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4290 {}
4291
4292 #else /* __lint */
4293
4294 #if defined(__amd64)
4295
4296 ENTRY(ftrace_interrupt_enable)
4297 pushq %rdi
4298 popfq
4299 ret
4300 SET_SIZE(ftrace_interrupt_enable)
4301
4302 #elif defined(__i386)
4303
4304 ENTRY(ftrace_interrupt_enable)
4305 movl 4(%esp), %eax
4306 pushl %eax
4307 popfl
4308 ret
4309 SET_SIZE(ftrace_interrupt_enable)
4310
4311 #endif /* __i386 */
4312 #endif /* __lint */
4313
4314 #if defined (__lint)
4315
4316 /*ARGSUSED*/
4317 void
4318 clflush_insn(caddr_t addr)
4319 {}
4320
4321 #else /* __lint */
4322
4323 #if defined (__amd64)
4324 ENTRY(clflush_insn)
4325 clflush (%rdi)
4326 ret
4327 SET_SIZE(clflush_insn)
4328 #elif defined (__i386)
4329 ENTRY(clflush_insn)
4330 movl 4(%esp), %eax
4331 clflush (%eax)
4332 ret
4333 SET_SIZE(clflush_insn)
4334
4335 #endif /* __i386 */
4336 #endif /* __lint */
4337
4338 #if defined (__lint)
4339 /*ARGSUSED*/
4340 void
4341 mfence_insn(void)
4342 {}
4343
4344 #else /* __lint */
4345
4346 #if defined (__amd64)
4347 ENTRY(mfence_insn)
4348 mfence
4349 ret
4350 SET_SIZE(mfence_insn)
4351 #elif defined (__i386)
4352 ENTRY(mfence_insn)
4353 mfence
4354 ret
4355 SET_SIZE(mfence_insn)
4356
4357 #endif /* __i386 */
4358 #endif /* __lint */
4359
4360 /*
4361 * VMware implements an I/O port that programs can query to detect if software
4362 * is running in a VMware hypervisor. This hypervisor port behaves differently
4363 * depending on magic values in certain registers and modifies some registers
4364 * as a side effect.
4365 *
4366 * References: http://kb.vmware.com/kb/1009458
4367 */
4368
4369 #if defined(__lint)
4370
4371 /* ARGSUSED */
4372 void
4373 vmware_port(int cmd, uint32_t *regs) { return; }
4374
4375 #else
4376
4377 #if defined(__amd64)
4378
4379 ENTRY(vmware_port)
4380 pushq %rbx
4381 movl $VMWARE_HVMAGIC, %eax
4382 movl $0xffffffff, %ebx
4383 movl %edi, %ecx
4384 movl $VMWARE_HVPORT, %edx
4385 inl (%dx)
4386 movl %eax, (%rsi)
4387 movl %ebx, 4(%rsi)
4388 movl %ecx, 8(%rsi)
4389 movl %edx, 12(%rsi)
4390 popq %rbx
4391 ret
4392 SET_SIZE(vmware_port)
4393
4394 #elif defined(__i386)
4395
4396 ENTRY(vmware_port)
4397 pushl %ebx
4398 pushl %esi
4399 movl $VMWARE_HVMAGIC, %eax
4400 movl $0xffffffff, %ebx
4401 movl 12(%esp), %ecx
4402 movl $VMWARE_HVPORT, %edx
4403 inl (%dx)
4404 movl 16(%esp), %esi
4405 movl %eax, (%esi)
4406 movl %ebx, 4(%esi)
4407 movl %ecx, 8(%esi)
4408 movl %edx, 12(%esi)
4409 popl %esi
4410 popl %ebx
4411 ret
4412 SET_SIZE(vmware_port)
4413
4414 #endif /* __i386 */
4415 #endif /* __lint */