1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 * Copyright 2018 Joyent, Inc.
27 */
28
29 /*
30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 * All Rights Reserved
33 */
34
35 /*
36 * Copyright (c) 2009, Intel Corporation.
37 * All rights reserved.
38 */
39
40 /*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
57 #include <sys/psw.h>
58 #include <sys/x86_archext.h>
59
60 #if defined(__lint)
61 #include <sys/types.h>
62 #include <sys/systm.h>
63 #include <sys/thread.h>
64 #include <sys/archsystm.h>
65 #include <sys/byteorder.h>
66 #include <sys/dtrace.h>
67 #include <sys/ftrace.h>
68 #else /* __lint */
69 #include "assym.h"
70 #endif /* __lint */
71 #include <sys/dditypes.h>
72
73 /*
74 * on_fault()
75 *
76 * Catch lofault faults. Like setjmp except it returns one
77 * if code following causes uncorrectable fault. Turned off
78 * by calling no_fault(). Note that while under on_fault(),
79 * SMAP is disabled. For more information see
80 * uts/intel/ia32/ml/copy.s.
81 */
82
83 #if defined(__lint)
84
85 /* ARGSUSED */
86 int
87 on_fault(label_t *ljb)
88 { return (0); }
89
90 void
91 no_fault(void)
92 {}
93
94 #else /* __lint */
95
96 #if defined(__amd64)
97
98 ENTRY(on_fault)
99 movq %gs:CPU_THREAD, %rsi
100 leaq catch_fault(%rip), %rdx
101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
103 call smap_disable /* allow user accesses */
104 jmp setjmp /* let setjmp do the rest */
105
106 catch_fault:
107 movq %gs:CPU_THREAD, %rsi
108 movq T_ONFAULT(%rsi), %rdi /* address of save area */
109 xorl %eax, %eax
110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
112 call smap_enable /* disallow user accesses */
113 jmp longjmp /* let longjmp do the rest */
114 SET_SIZE(on_fault)
115
116 ENTRY(no_fault)
117 movq %gs:CPU_THREAD, %rsi
118 xorl %eax, %eax
119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
121 call smap_enable /* disallow user accesses */
122 ret
123 SET_SIZE(no_fault)
124
125 #elif defined(__i386)
126
127 ENTRY(on_fault)
128 movl %gs:CPU_THREAD, %edx
129 movl 4(%esp), %eax /* jumpbuf address */
130 leal catch_fault, %ecx
131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
133 jmp setjmp /* let setjmp do the rest */
134
135 catch_fault:
136 movl %gs:CPU_THREAD, %edx
137 xorl %eax, %eax
138 movl T_ONFAULT(%edx), %ecx /* address of save area */
139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
141 pushl %ecx
142 call longjmp /* let longjmp do the rest */
143 SET_SIZE(on_fault)
144
145 ENTRY(no_fault)
146 movl %gs:CPU_THREAD, %edx
147 xorl %eax, %eax
148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
150 ret
151 SET_SIZE(no_fault)
152
153 #endif /* __i386 */
154 #endif /* __lint */
155
156 /*
157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 */
160
161 #if defined(lint)
162
163 void
164 on_trap_trampoline(void)
165 {}
166
167 #else /* __lint */
168
169 #if defined(__amd64)
170
171 ENTRY(on_trap_trampoline)
172 movq %gs:CPU_THREAD, %rsi
173 movq T_ONTRAP(%rsi), %rdi
174 addq $OT_JMPBUF, %rdi
175 jmp longjmp
176 SET_SIZE(on_trap_trampoline)
177
178 #elif defined(__i386)
179
180 ENTRY(on_trap_trampoline)
181 movl %gs:CPU_THREAD, %eax
182 movl T_ONTRAP(%eax), %eax
183 addl $OT_JMPBUF, %eax
184 pushl %eax
185 call longjmp
186 SET_SIZE(on_trap_trampoline)
187
188 #endif /* __i386 */
189 #endif /* __lint */
190
191 /*
192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
193 * more information about the on_trap() mechanism. If the on_trap_data is the
194 * same as the topmost stack element, we just modify that element.
195 */
196 #if defined(lint)
197
198 /*ARGSUSED*/
199 int
200 on_trap(on_trap_data_t *otp, uint_t prot)
201 { return (0); }
202
203 #else /* __lint */
204
205 #if defined(__amd64)
206
207 ENTRY(on_trap)
208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
212 xorl %ecx, %ecx
213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
217 cmpq %rdi, %rcx /* if (otp == %rcx) */
218 je 0f /* don't modify t_ontrap */
219
220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
222
223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
224 jmp setjmp
225 SET_SIZE(on_trap)
226
227 #elif defined(__i386)
228
229 ENTRY(on_trap)
230 movl 4(%esp), %eax /* %eax = otp */
231 movl 8(%esp), %edx /* %edx = prot */
232
233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
241 cmpl %eax, %ecx /* if (otp == %ecx) */
242 je 0f /* don't modify t_ontrap */
243
244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
246
247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
248 movl %eax, 4(%esp) /* put %eax back on the stack */
249 jmp setjmp /* let setjmp do the rest */
250 SET_SIZE(on_trap)
251
252 #endif /* __i386 */
253 #endif /* __lint */
254
255 /*
256 * Setjmp and longjmp implement non-local gotos using state vectors
257 * type label_t.
258 */
259
260 #if defined(__lint)
261
262 /* ARGSUSED */
263 int
264 setjmp(label_t *lp)
265 { return (0); }
266
267 /* ARGSUSED */
268 void
269 longjmp(label_t *lp)
270 {}
271
272 #else /* __lint */
273
274 #if LABEL_PC != 0
275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276 #endif /* LABEL_PC != 0 */
277
278 #if defined(__amd64)
279
280 ENTRY(setjmp)
281 movq %rsp, LABEL_SP(%rdi)
282 movq %rbp, LABEL_RBP(%rdi)
283 movq %rbx, LABEL_RBX(%rdi)
284 movq %r12, LABEL_R12(%rdi)
285 movq %r13, LABEL_R13(%rdi)
286 movq %r14, LABEL_R14(%rdi)
287 movq %r15, LABEL_R15(%rdi)
288 movq (%rsp), %rdx /* return address */
289 movq %rdx, (%rdi) /* LABEL_PC is 0 */
290 xorl %eax, %eax /* return 0 */
291 ret
292 SET_SIZE(setjmp)
293
294 ENTRY(longjmp)
295 movq LABEL_SP(%rdi), %rsp
296 movq LABEL_RBP(%rdi), %rbp
297 movq LABEL_RBX(%rdi), %rbx
298 movq LABEL_R12(%rdi), %r12
299 movq LABEL_R13(%rdi), %r13
300 movq LABEL_R14(%rdi), %r14
301 movq LABEL_R15(%rdi), %r15
302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
303 movq %rdx, (%rsp)
304 xorl %eax, %eax
305 incl %eax /* return 1 */
306 ret
307 SET_SIZE(longjmp)
308
309 #elif defined(__i386)
310
311 ENTRY(setjmp)
312 movl 4(%esp), %edx /* address of save area */
313 movl %ebp, LABEL_EBP(%edx)
314 movl %ebx, LABEL_EBX(%edx)
315 movl %esi, LABEL_ESI(%edx)
316 movl %edi, LABEL_EDI(%edx)
317 movl %esp, 4(%edx)
318 movl (%esp), %ecx /* %eip (return address) */
319 movl %ecx, (%edx) /* LABEL_PC is 0 */
320 subl %eax, %eax /* return 0 */
321 ret
322 SET_SIZE(setjmp)
323
324 ENTRY(longjmp)
325 movl 4(%esp), %edx /* address of save area */
326 movl LABEL_EBP(%edx), %ebp
327 movl LABEL_EBX(%edx), %ebx
328 movl LABEL_ESI(%edx), %esi
329 movl LABEL_EDI(%edx), %edi
330 movl 4(%edx), %esp
331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
332 movl $1, %eax
333 addl $4, %esp /* pop ret adr */
334 jmp *%ecx /* indirect */
335 SET_SIZE(longjmp)
336
337 #endif /* __i386 */
338 #endif /* __lint */
339
340 /*
341 * if a() calls b() calls caller(),
342 * caller() returns return address in a().
343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 * sequence.)
345 */
346
347 #if defined(__lint)
348
349 caddr_t
350 caller(void)
351 { return (0); }
352
353 #else /* __lint */
354
355 #if defined(__amd64)
356
357 ENTRY(caller)
358 movq 8(%rbp), %rax /* b()'s return pc, in a() */
359 ret
360 SET_SIZE(caller)
361
362 #elif defined(__i386)
363
364 ENTRY(caller)
365 movl 4(%ebp), %eax /* b()'s return pc, in a() */
366 ret
367 SET_SIZE(caller)
368
369 #endif /* __i386 */
370 #endif /* __lint */
371
372 /*
373 * if a() calls callee(), callee() returns the
374 * return address in a();
375 */
376
377 #if defined(__lint)
378
379 caddr_t
380 callee(void)
381 { return (0); }
382
383 #else /* __lint */
384
385 #if defined(__amd64)
386
387 ENTRY(callee)
388 movq (%rsp), %rax /* callee()'s return pc, in a() */
389 ret
390 SET_SIZE(callee)
391
392 #elif defined(__i386)
393
394 ENTRY(callee)
395 movl (%esp), %eax /* callee()'s return pc, in a() */
396 ret
397 SET_SIZE(callee)
398
399 #endif /* __i386 */
400 #endif /* __lint */
401
402 /*
403 * return the current frame pointer
404 */
405
406 #if defined(__lint)
407
408 greg_t
409 getfp(void)
410 { return (0); }
411
412 #else /* __lint */
413
414 #if defined(__amd64)
415
416 ENTRY(getfp)
417 movq %rbp, %rax
418 ret
419 SET_SIZE(getfp)
420
421 #elif defined(__i386)
422
423 ENTRY(getfp)
424 movl %ebp, %eax
425 ret
426 SET_SIZE(getfp)
427
428 #endif /* __i386 */
429 #endif /* __lint */
430
431 /*
432 * Invalidate a single page table entry in the TLB
433 */
434
435 #if defined(__lint)
436
437 /* ARGSUSED */
438 void
439 mmu_invlpg(caddr_t m)
440 {}
441
442 #else /* __lint */
443
444 ENTRY(mmu_invlpg)
445 invlpg (%rdi)
446 ret
447 SET_SIZE(mmu_invlpg)
448
449 #endif /* __lint */
450
451
452 /*
453 * Get/Set the value of various control registers
454 */
455
456 #if defined(__lint)
457
458 ulong_t
459 getcr0(void)
460 { return (0); }
461
462 /* ARGSUSED */
463 void
464 setcr0(ulong_t value)
465 {}
466
467 ulong_t
468 getcr2(void)
469 { return (0); }
470
471 ulong_t
472 getcr3(void)
473 { return (0); }
474
475 #if !defined(__xpv)
476 /* ARGSUSED */
477 void
478 setcr3(ulong_t val)
479 {}
480
481 void
482 reload_cr3(void)
483 {}
484 #endif
485
486 ulong_t
487 getcr4(void)
488 { return (0); }
489
490 /* ARGSUSED */
491 void
492 setcr4(ulong_t val)
493 {}
494
495 #if defined(__amd64)
496
497 ulong_t
498 getcr8(void)
499 { return (0); }
500
501 /* ARGSUSED */
502 void
503 setcr8(ulong_t val)
504 {}
505
506 #endif /* __amd64 */
507
508 #else /* __lint */
509
510 #if defined(__amd64)
511
512 ENTRY(getcr0)
513 movq %cr0, %rax
514 ret
515 SET_SIZE(getcr0)
516
517 ENTRY(setcr0)
518 movq %rdi, %cr0
519 ret
520 SET_SIZE(setcr0)
521
522 ENTRY(getcr2)
523 #if defined(__xpv)
524 movq %gs:CPU_VCPU_INFO, %rax
525 movq VCPU_INFO_ARCH_CR2(%rax), %rax
526 #else
527 movq %cr2, %rax
528 #endif
529 ret
530 SET_SIZE(getcr2)
531
532 ENTRY(getcr3)
533 movq %cr3, %rax
534 ret
535 SET_SIZE(getcr3)
536
537 #if !defined(__xpv)
538
539 ENTRY(setcr3)
540 movq %rdi, %cr3
541 ret
542 SET_SIZE(setcr3)
543
544 ENTRY(reload_cr3)
545 movq %cr3, %rdi
546 movq %rdi, %cr3
547 ret
548 SET_SIZE(reload_cr3)
549
550 #endif /* __xpv */
551
552 ENTRY(getcr4)
553 movq %cr4, %rax
554 ret
555 SET_SIZE(getcr4)
556
557 ENTRY(setcr4)
558 movq %rdi, %cr4
559 ret
560 SET_SIZE(setcr4)
561
562 ENTRY(getcr8)
563 movq %cr8, %rax
564 ret
565 SET_SIZE(getcr8)
566
567 ENTRY(setcr8)
568 movq %rdi, %cr8
569 ret
570 SET_SIZE(setcr8)
571
572 #elif defined(__i386)
573
574 ENTRY(getcr0)
575 movl %cr0, %eax
576 ret
577 SET_SIZE(getcr0)
578
579 ENTRY(setcr0)
580 movl 4(%esp), %eax
581 movl %eax, %cr0
582 ret
583 SET_SIZE(setcr0)
584
585 /*
586 * "lock mov %cr0" is used on processors which indicate it is
587 * supported via CPUID. Normally the 32 bit TPR is accessed via
588 * the local APIC.
589 */
590 ENTRY(getcr8)
591 lock
592 movl %cr0, %eax
593 ret
594 SET_SIZE(getcr8)
595
596 ENTRY(setcr8)
597 movl 4(%esp), %eax
598 lock
599 movl %eax, %cr0
600 ret
601 SET_SIZE(setcr8)
602
603 ENTRY(getcr2)
604 #if defined(__xpv)
605 movl %gs:CPU_VCPU_INFO, %eax
606 movl VCPU_INFO_ARCH_CR2(%eax), %eax
607 #else
608 movl %cr2, %eax
609 #endif
610 ret
611 SET_SIZE(getcr2)
612
613 ENTRY(getcr3)
614 movl %cr3, %eax
615 ret
616 SET_SIZE(getcr3)
617
618 #if !defined(__xpv)
619
620 ENTRY(setcr3)
621 movl 4(%esp), %eax
622 movl %eax, %cr3
623 ret
624 SET_SIZE(setcr3)
625
626 ENTRY(reload_cr3)
627 movl %cr3, %eax
628 movl %eax, %cr3
629 ret
630 SET_SIZE(reload_cr3)
631
632 #endif /* __xpv */
633
634 ENTRY(getcr4)
635 movl %cr4, %eax
636 ret
637 SET_SIZE(getcr4)
638
639 ENTRY(setcr4)
640 movl 4(%esp), %eax
641 movl %eax, %cr4
642 ret
643 SET_SIZE(setcr4)
644
645 #endif /* __i386 */
646 #endif /* __lint */
647
648 #if defined(__lint)
649
650 /*ARGSUSED*/
651 uint32_t
652 __cpuid_insn(struct cpuid_regs *regs)
653 { return (0); }
654
655 #else /* __lint */
656
657 #if defined(__amd64)
658
659 ENTRY(__cpuid_insn)
660 movq %rbx, %r8
661 movq %rcx, %r9
662 movq %rdx, %r11
663 movl (%rdi), %eax /* %eax = regs->cp_eax */
664 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
665 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
666 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
667 cpuid
668 movl %eax, (%rdi) /* regs->cp_eax = %eax */
669 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
670 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
671 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
672 movq %r8, %rbx
673 movq %r9, %rcx
674 movq %r11, %rdx
675 ret
676 SET_SIZE(__cpuid_insn)
677
678 #elif defined(__i386)
679
680 ENTRY(__cpuid_insn)
681 pushl %ebp
682 movl 0x8(%esp), %ebp /* %ebp = regs */
683 pushl %ebx
684 pushl %ecx
685 pushl %edx
686 movl (%ebp), %eax /* %eax = regs->cp_eax */
687 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
688 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
689 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
690 cpuid
691 movl %eax, (%ebp) /* regs->cp_eax = %eax */
692 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
693 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
694 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
695 popl %edx
696 popl %ecx
697 popl %ebx
698 popl %ebp
699 ret
700 SET_SIZE(__cpuid_insn)
701
702 #endif /* __i386 */
703 #endif /* __lint */
704
705 #if defined(__lint)
706
707 /*ARGSUSED*/
708 void
709 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
710 {}
711
712 #else /* __lint */
713
714 #if defined(__amd64)
715
716 ENTRY_NP(i86_monitor)
717 pushq %rbp
718 movq %rsp, %rbp
719 movq %rdi, %rax /* addr */
720 movq %rsi, %rcx /* extensions */
721 /* rdx contains input arg3: hints */
722 clflush (%rax)
723 .byte 0x0f, 0x01, 0xc8 /* monitor */
724 leave
725 ret
726 SET_SIZE(i86_monitor)
727
728 #elif defined(__i386)
729
730 ENTRY_NP(i86_monitor)
731 pushl %ebp
732 movl %esp, %ebp
733 movl 0x8(%ebp),%eax /* addr */
734 movl 0xc(%ebp),%ecx /* extensions */
735 movl 0x10(%ebp),%edx /* hints */
736 clflush (%eax)
737 .byte 0x0f, 0x01, 0xc8 /* monitor */
738 leave
739 ret
740 SET_SIZE(i86_monitor)
741
742 #endif /* __i386 */
743 #endif /* __lint */
744
745 #if defined(__lint)
746
747 /*ARGSUSED*/
748 void
749 i86_mwait(uint32_t data, uint32_t extensions)
750 {}
751
752 #else /* __lint */
753
754 #if defined(__amd64)
755
756 ENTRY_NP(i86_mwait)
757 pushq %rbp
758 movq %rsp, %rbp
759 movq %rdi, %rax /* data */
760 movq %rsi, %rcx /* extensions */
761 .byte 0x0f, 0x01, 0xc9 /* mwait */
762 leave
763 ret
764 SET_SIZE(i86_mwait)
765
766 #elif defined(__i386)
767
768 ENTRY_NP(i86_mwait)
769 pushl %ebp
770 movl %esp, %ebp
771 movl 0x8(%ebp),%eax /* data */
772 movl 0xc(%ebp),%ecx /* extensions */
773 .byte 0x0f, 0x01, 0xc9 /* mwait */
774 leave
775 ret
776 SET_SIZE(i86_mwait)
777
778 #endif /* __i386 */
779 #endif /* __lint */
780
781 #if defined(__xpv)
782 /*
783 * Defined in C
784 */
785 #else
786
787 #if defined(__lint)
788
789 hrtime_t
790 tsc_read(void)
791 {
792 return (0);
793 }
794
795 #else /* __lint */
796
797 #if defined(__amd64)
798
799 ENTRY_NP(tsc_read)
800 movq %rbx, %r11
801 movl $0, %eax
802 cpuid
803 rdtsc
804 movq %r11, %rbx
805 shlq $32, %rdx
806 orq %rdx, %rax
807 ret
808 .globl _tsc_mfence_start
809 _tsc_mfence_start:
810 mfence
811 rdtsc
812 shlq $32, %rdx
813 orq %rdx, %rax
814 ret
815 .globl _tsc_mfence_end
816 _tsc_mfence_end:
817 .globl _tscp_start
818 _tscp_start:
819 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
820 shlq $32, %rdx
821 orq %rdx, %rax
822 ret
823 .globl _tscp_end
824 _tscp_end:
825 .globl _no_rdtsc_start
826 _no_rdtsc_start:
827 xorl %edx, %edx
828 xorl %eax, %eax
829 ret
830 .globl _no_rdtsc_end
831 _no_rdtsc_end:
832 .globl _tsc_lfence_start
833 _tsc_lfence_start:
834 lfence
835 rdtsc
836 shlq $32, %rdx
837 orq %rdx, %rax
838 ret
839 .globl _tsc_lfence_end
840 _tsc_lfence_end:
841 SET_SIZE(tsc_read)
842
843 #else /* __i386 */
844
845 ENTRY_NP(tsc_read)
846 pushl %ebx
847 movl $0, %eax
848 cpuid
849 rdtsc
850 popl %ebx
851 ret
852 .globl _tsc_mfence_start
853 _tsc_mfence_start:
854 mfence
855 rdtsc
856 ret
857 .globl _tsc_mfence_end
858 _tsc_mfence_end:
859 .globl _tscp_start
860 _tscp_start:
861 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
862 ret
863 .globl _tscp_end
864 _tscp_end:
865 .globl _no_rdtsc_start
866 _no_rdtsc_start:
867 xorl %edx, %edx
868 xorl %eax, %eax
869 ret
870 .globl _no_rdtsc_end
871 _no_rdtsc_end:
872 .globl _tsc_lfence_start
873 _tsc_lfence_start:
874 lfence
875 rdtsc
876 ret
877 .globl _tsc_lfence_end
878 _tsc_lfence_end:
879 SET_SIZE(tsc_read)
880
881 #endif /* __i386 */
882
883 #endif /* __lint */
884
885
886 #endif /* __xpv */
887
888 #ifdef __lint
889 /*
890 * Do not use this function for obtaining clock tick. This
891 * is called by callers who do not need to have a guarenteed
892 * correct tick value. The proper routine to use is tsc_read().
893 */
894 u_longlong_t
895 randtick(void)
896 {
897 return (0);
898 }
899 #else
900 #if defined(__amd64)
901 ENTRY_NP(randtick)
902 rdtsc
903 shlq $32, %rdx
904 orq %rdx, %rax
905 ret
906 SET_SIZE(randtick)
907 #else
908 ENTRY_NP(randtick)
909 rdtsc
910 ret
911 SET_SIZE(randtick)
912 #endif /* __i386 */
913 #endif /* __lint */
914 /*
915 * Insert entryp after predp in a doubly linked list.
916 */
917
918 #if defined(__lint)
919
920 /*ARGSUSED*/
921 void
922 _insque(caddr_t entryp, caddr_t predp)
923 {}
924
925 #else /* __lint */
926
927 #if defined(__amd64)
928
929 ENTRY(_insque)
930 movq (%rsi), %rax /* predp->forw */
931 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
932 movq %rax, (%rdi) /* entryp->forw = predp->forw */
933 movq %rdi, (%rsi) /* predp->forw = entryp */
934 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
935 ret
936 SET_SIZE(_insque)
937
938 #elif defined(__i386)
939
940 ENTRY(_insque)
941 movl 8(%esp), %edx
942 movl 4(%esp), %ecx
943 movl (%edx), %eax /* predp->forw */
944 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
945 movl %eax, (%ecx) /* entryp->forw = predp->forw */
946 movl %ecx, (%edx) /* predp->forw = entryp */
947 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
948 ret
949 SET_SIZE(_insque)
950
951 #endif /* __i386 */
952 #endif /* __lint */
953
954 /*
955 * Remove entryp from a doubly linked list
956 */
957
958 #if defined(__lint)
959
960 /*ARGSUSED*/
961 void
962 _remque(caddr_t entryp)
963 {}
964
965 #else /* __lint */
966
967 #if defined(__amd64)
968
969 ENTRY(_remque)
970 movq (%rdi), %rax /* entry->forw */
971 movq CPTRSIZE(%rdi), %rdx /* entry->back */
972 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
973 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
974 ret
975 SET_SIZE(_remque)
976
977 #elif defined(__i386)
978
979 ENTRY(_remque)
980 movl 4(%esp), %ecx
981 movl (%ecx), %eax /* entry->forw */
982 movl CPTRSIZE(%ecx), %edx /* entry->back */
983 movl %eax, (%edx) /* entry->back->forw = entry->forw */
984 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
985 ret
986 SET_SIZE(_remque)
987
988 #endif /* __i386 */
989 #endif /* __lint */
990
991 /*
992 * Returns the number of
993 * non-NULL bytes in string argument.
994 */
995
996 #if defined(__lint)
997
998 /* ARGSUSED */
999 size_t
1000 strlen(const char *str)
1001 { return (0); }
1002
1003 #else /* __lint */
1004
1005 #if defined(__amd64)
1006
1007 /*
1008 * This is close to a simple transliteration of a C version of this
1009 * routine. We should either just -make- this be a C version, or
1010 * justify having it in assembler by making it significantly faster.
1011 *
1012 * size_t
1013 * strlen(const char *s)
1014 * {
1015 * const char *s0;
1016 * #if defined(DEBUG)
1017 * if ((uintptr_t)s < KERNELBASE)
1018 * panic(.str_panic_msg);
1019 * #endif
1020 * for (s0 = s; *s; s++)
1021 * ;
1022 * return (s - s0);
1023 * }
1024 */
1025
1026 ENTRY(strlen)
1027 #ifdef DEBUG
1028 movq postbootkernelbase(%rip), %rax
1029 cmpq %rax, %rdi
1030 jae str_valid
1031 pushq %rbp
1032 movq %rsp, %rbp
1033 leaq .str_panic_msg(%rip), %rdi
1034 xorl %eax, %eax
1035 call panic
1036 #endif /* DEBUG */
1037 str_valid:
1038 cmpb $0, (%rdi)
1039 movq %rdi, %rax
1040 je .null_found
1041 .align 4
1042 .strlen_loop:
1043 incq %rdi
1044 cmpb $0, (%rdi)
1045 jne .strlen_loop
1046 .null_found:
1047 subq %rax, %rdi
1048 movq %rdi, %rax
1049 ret
1050 SET_SIZE(strlen)
1051
1052 #elif defined(__i386)
1053
1054 ENTRY(strlen)
1055 #ifdef DEBUG
1056 movl postbootkernelbase, %eax
1057 cmpl %eax, 4(%esp)
1058 jae str_valid
1059 pushl %ebp
1060 movl %esp, %ebp
1061 pushl $.str_panic_msg
1062 call panic
1063 #endif /* DEBUG */
1064
1065 str_valid:
1066 movl 4(%esp), %eax /* %eax = string address */
1067 testl $3, %eax /* if %eax not word aligned */
1068 jnz .not_word_aligned /* goto .not_word_aligned */
1069 .align 4
1070 .word_aligned:
1071 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1072 movl $0x7f7f7f7f, %ecx
1073 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1074 addl $4, %eax /* next word */
1075 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1076 orl %edx, %ecx /* %ecx |= %edx */
1077 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1078 cmpl $0x80808080, %ecx /* if no null byte in this word */
1079 je .word_aligned /* goto .word_aligned */
1080 subl $4, %eax /* post-incremented */
1081 .not_word_aligned:
1082 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1083 je .null_found /* goto .null_found */
1084 incl %eax /* next byte */
1085 testl $3, %eax /* if %eax not word aligned */
1086 jnz .not_word_aligned /* goto .not_word_aligned */
1087 jmp .word_aligned /* goto .word_aligned */
1088 .align 4
1089 .null_found:
1090 subl 4(%esp), %eax /* %eax -= string address */
1091 ret
1092 SET_SIZE(strlen)
1093
1094 #endif /* __i386 */
1095
1096 #ifdef DEBUG
1097 .text
1098 .str_panic_msg:
1099 .string "strlen: argument below kernelbase"
1100 #endif /* DEBUG */
1101
1102 #endif /* __lint */
1103
1104 /*
1105 * Berkeley 4.3 introduced symbolically named interrupt levels
1106 * as a way deal with priority in a machine independent fashion.
1107 * Numbered priorities are machine specific, and should be
1108 * discouraged where possible.
1109 *
1110 * Note, for the machine specific priorities there are
1111 * examples listed for devices that use a particular priority.
1112 * It should not be construed that all devices of that
1113 * type should be at that priority. It is currently were
1114 * the current devices fit into the priority scheme based
1115 * upon time criticalness.
1116 *
1117 * The underlying assumption of these assignments is that
1118 * IPL 10 is the highest level from which a device
1119 * routine can call wakeup. Devices that interrupt from higher
1120 * levels are restricted in what they can do. If they need
1121 * kernels services they should schedule a routine at a lower
1122 * level (via software interrupt) to do the required
1123 * processing.
1124 *
1125 * Examples of this higher usage:
1126 * Level Usage
1127 * 14 Profiling clock (and PROM uart polling clock)
1128 * 12 Serial ports
1129 *
1130 * The serial ports request lower level processing on level 6.
1131 *
1132 * Also, almost all splN routines (where N is a number or a
1133 * mnemonic) will do a RAISE(), on the assumption that they are
1134 * never used to lower our priority.
1135 * The exceptions are:
1136 * spl8() Because you can't be above 15 to begin with!
1137 * splzs() Because this is used at boot time to lower our
1138 * priority, to allow the PROM to poll the uart.
1139 * spl0() Used to lower priority to 0.
1140 */
1141
1142 #if defined(__lint)
1143
1144 int spl0(void) { return (0); }
1145 int spl6(void) { return (0); }
1146 int spl7(void) { return (0); }
1147 int spl8(void) { return (0); }
1148 int splhigh(void) { return (0); }
1149 int splhi(void) { return (0); }
1150 int splzs(void) { return (0); }
1151
1152 /* ARGSUSED */
1153 void
1154 splx(int level)
1155 {}
1156
1157 #else /* __lint */
1158
1159 #if defined(__amd64)
1160
1161 #define SETPRI(level) \
1162 movl $/**/level, %edi; /* new priority */ \
1163 jmp do_splx /* redirect to do_splx */
1164
1165 #define RAISE(level) \
1166 movl $/**/level, %edi; /* new priority */ \
1167 jmp splr /* redirect to splr */
1168
1169 #elif defined(__i386)
1170
1171 #define SETPRI(level) \
1172 pushl $/**/level; /* new priority */ \
1173 call do_splx; /* invoke common splx code */ \
1174 addl $4, %esp; /* unstack arg */ \
1175 ret
1176
1177 #define RAISE(level) \
1178 pushl $/**/level; /* new priority */ \
1179 call splr; /* invoke common splr code */ \
1180 addl $4, %esp; /* unstack args */ \
1181 ret
1182
1183 #endif /* __i386 */
1184
1185 /* locks out all interrupts, including memory errors */
1186 ENTRY(spl8)
1187 SETPRI(15)
1188 SET_SIZE(spl8)
1189
1190 /* just below the level that profiling runs */
1191 ENTRY(spl7)
1192 RAISE(13)
1193 SET_SIZE(spl7)
1194
1195 /* sun specific - highest priority onboard serial i/o asy ports */
1196 ENTRY(splzs)
1197 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1198 SET_SIZE(splzs)
1199
1200 ENTRY(splhi)
1201 ALTENTRY(splhigh)
1202 ALTENTRY(spl6)
1203 ALTENTRY(i_ddi_splhigh)
1204
1205 RAISE(DISP_LEVEL)
1206
1207 SET_SIZE(i_ddi_splhigh)
1208 SET_SIZE(spl6)
1209 SET_SIZE(splhigh)
1210 SET_SIZE(splhi)
1211
1212 /* allow all interrupts */
1213 ENTRY(spl0)
1214 SETPRI(0)
1215 SET_SIZE(spl0)
1216
1217
1218 /* splx implementation */
1219 ENTRY(splx)
1220 jmp do_splx /* redirect to common splx code */
1221 SET_SIZE(splx)
1222
1223 #endif /* __lint */
1224
1225 #if defined(__i386)
1226
1227 /*
1228 * Read and write the %gs register
1229 */
1230
1231 #if defined(__lint)
1232
1233 /*ARGSUSED*/
1234 uint16_t
1235 getgs(void)
1236 { return (0); }
1237
1238 /*ARGSUSED*/
1239 void
1240 setgs(uint16_t sel)
1241 {}
1242
1243 #else /* __lint */
1244
1245 ENTRY(getgs)
1246 clr %eax
1247 movw %gs, %ax
1248 ret
1249 SET_SIZE(getgs)
1250
1251 ENTRY(setgs)
1252 movw 4(%esp), %gs
1253 ret
1254 SET_SIZE(setgs)
1255
1256 #endif /* __lint */
1257 #endif /* __i386 */
1258
1259 #if defined(__lint)
1260
1261 void
1262 pc_reset(void)
1263 {}
1264
1265 void
1266 efi_reset(void)
1267 {}
1268
1269 #else /* __lint */
1270
1271 ENTRY(wait_500ms)
1272 #if defined(__amd64)
1273 pushq %rbx
1274 #elif defined(__i386)
1275 push %ebx
1276 #endif
1277 movl $50000, %ebx
1278 1:
1279 call tenmicrosec
1280 decl %ebx
1281 jnz 1b
1282 #if defined(__amd64)
1283 popq %rbx
1284 #elif defined(__i386)
1285 pop %ebx
1286 #endif
1287 ret
1288 SET_SIZE(wait_500ms)
1289
1290 #define RESET_METHOD_KBC 1
1291 #define RESET_METHOD_PORT92 2
1292 #define RESET_METHOD_PCI 4
1293
1294 DGDEF3(pc_reset_methods, 4, 8)
1295 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1296
1297 ENTRY(pc_reset)
1298
1299 #if defined(__i386)
1300 testl $RESET_METHOD_KBC, pc_reset_methods
1301 #elif defined(__amd64)
1302 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1303 #endif
1304 jz 1f
1305
1306 /
1307 / Try the classic keyboard controller-triggered reset.
1308 /
1309 movw $0x64, %dx
1310 movb $0xfe, %al
1311 outb (%dx)
1312
1313 / Wait up to 500 milliseconds here for the keyboard controller
1314 / to pull the reset line. On some systems where the keyboard
1315 / controller is slow to pull the reset line, the next reset method
1316 / may be executed (which may be bad if those systems hang when the
1317 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1318 / and Ferrari 4000 (doesn't like the cf9 reset method))
1319
1320 call wait_500ms
1321
1322 1:
1323 #if defined(__i386)
1324 testl $RESET_METHOD_PORT92, pc_reset_methods
1325 #elif defined(__amd64)
1326 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1327 #endif
1328 jz 3f
1329
1330 /
1331 / Try port 0x92 fast reset
1332 /
1333 movw $0x92, %dx
1334 inb (%dx)
1335 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1336 je 1f
1337 testb $1, %al / If bit 0
1338 jz 2f / is clear, jump to perform the reset
1339 andb $0xfe, %al / otherwise,
1340 outb (%dx) / clear bit 0 first, then
1341 2:
1342 orb $1, %al / Set bit 0
1343 outb (%dx) / and reset the system
1344 1:
1345
1346 call wait_500ms
1347
1348 3:
1349 #if defined(__i386)
1350 testl $RESET_METHOD_PCI, pc_reset_methods
1351 #elif defined(__amd64)
1352 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1353 #endif
1354 jz 4f
1355
1356 / Try the PCI (soft) reset vector (should work on all modern systems,
1357 / but has been shown to cause problems on 450NX systems, and some newer
1358 / systems (e.g. ATI IXP400-equipped systems))
1359 / When resetting via this method, 2 writes are required. The first
1360 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1361 / power cycle).
1362 / The reset occurs on the second write, during bit 2's transition from
1363 / 0->1.
1364 movw $0xcf9, %dx
1365 movb $0x2, %al / Reset mode = hard, no power cycle
1366 outb (%dx)
1367 movb $0x6, %al
1368 outb (%dx)
1369
1370 call wait_500ms
1371
1372 4:
1373 /
1374 / port 0xcf9 failed also. Last-ditch effort is to
1375 / triple-fault the CPU.
1376 / Also, use triple fault for EFI firmware
1377 /
1378 ENTRY(efi_reset)
1379 #if defined(__amd64)
1380 pushq $0x0
1381 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1382 lidt (%rsp)
1383 #elif defined(__i386)
1384 pushl $0x0
1385 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1386 lidt (%esp)
1387 #endif
1388 int $0x0 / Trigger interrupt, generate triple-fault
1389
1390 cli
1391 hlt / Wait forever
1392 /*NOTREACHED*/
1393 SET_SIZE(efi_reset)
1394 SET_SIZE(pc_reset)
1395
1396 #endif /* __lint */
1397
1398 /*
1399 * C callable in and out routines
1400 */
1401
1402 #if defined(__lint)
1403
1404 /* ARGSUSED */
1405 void
1406 outl(int port_address, uint32_t val)
1407 {}
1408
1409 #else /* __lint */
1410
1411 #if defined(__amd64)
1412
1413 ENTRY(outl)
1414 movw %di, %dx
1415 movl %esi, %eax
1416 outl (%dx)
1417 ret
1418 SET_SIZE(outl)
1419
1420 #elif defined(__i386)
1421
1422 .set PORT, 4
1423 .set VAL, 8
1424
1425 ENTRY(outl)
1426 movw PORT(%esp), %dx
1427 movl VAL(%esp), %eax
1428 outl (%dx)
1429 ret
1430 SET_SIZE(outl)
1431
1432 #endif /* __i386 */
1433 #endif /* __lint */
1434
1435 #if defined(__lint)
1436
1437 /* ARGSUSED */
1438 void
1439 outw(int port_address, uint16_t val)
1440 {}
1441
1442 #else /* __lint */
1443
1444 #if defined(__amd64)
1445
1446 ENTRY(outw)
1447 movw %di, %dx
1448 movw %si, %ax
1449 D16 outl (%dx) /* XX64 why not outw? */
1450 ret
1451 SET_SIZE(outw)
1452
1453 #elif defined(__i386)
1454
1455 ENTRY(outw)
1456 movw PORT(%esp), %dx
1457 movw VAL(%esp), %ax
1458 D16 outl (%dx)
1459 ret
1460 SET_SIZE(outw)
1461
1462 #endif /* __i386 */
1463 #endif /* __lint */
1464
1465 #if defined(__lint)
1466
1467 /* ARGSUSED */
1468 void
1469 outb(int port_address, uint8_t val)
1470 {}
1471
1472 #else /* __lint */
1473
1474 #if defined(__amd64)
1475
1476 ENTRY(outb)
1477 movw %di, %dx
1478 movb %sil, %al
1479 outb (%dx)
1480 ret
1481 SET_SIZE(outb)
1482
1483 #elif defined(__i386)
1484
1485 ENTRY(outb)
1486 movw PORT(%esp), %dx
1487 movb VAL(%esp), %al
1488 outb (%dx)
1489 ret
1490 SET_SIZE(outb)
1491
1492 #endif /* __i386 */
1493 #endif /* __lint */
1494
1495 #if defined(__lint)
1496
1497 /* ARGSUSED */
1498 uint32_t
1499 inl(int port_address)
1500 { return (0); }
1501
1502 #else /* __lint */
1503
1504 #if defined(__amd64)
1505
1506 ENTRY(inl)
1507 xorl %eax, %eax
1508 movw %di, %dx
1509 inl (%dx)
1510 ret
1511 SET_SIZE(inl)
1512
1513 #elif defined(__i386)
1514
1515 ENTRY(inl)
1516 movw PORT(%esp), %dx
1517 inl (%dx)
1518 ret
1519 SET_SIZE(inl)
1520
1521 #endif /* __i386 */
1522 #endif /* __lint */
1523
1524 #if defined(__lint)
1525
1526 /* ARGSUSED */
1527 uint16_t
1528 inw(int port_address)
1529 { return (0); }
1530
1531 #else /* __lint */
1532
1533 #if defined(__amd64)
1534
1535 ENTRY(inw)
1536 xorl %eax, %eax
1537 movw %di, %dx
1538 D16 inl (%dx)
1539 ret
1540 SET_SIZE(inw)
1541
1542 #elif defined(__i386)
1543
1544 ENTRY(inw)
1545 subl %eax, %eax
1546 movw PORT(%esp), %dx
1547 D16 inl (%dx)
1548 ret
1549 SET_SIZE(inw)
1550
1551 #endif /* __i386 */
1552 #endif /* __lint */
1553
1554
1555 #if defined(__lint)
1556
1557 /* ARGSUSED */
1558 uint8_t
1559 inb(int port_address)
1560 { return (0); }
1561
1562 #else /* __lint */
1563
1564 #if defined(__amd64)
1565
1566 ENTRY(inb)
1567 xorl %eax, %eax
1568 movw %di, %dx
1569 inb (%dx)
1570 ret
1571 SET_SIZE(inb)
1572
1573 #elif defined(__i386)
1574
1575 ENTRY(inb)
1576 subl %eax, %eax
1577 movw PORT(%esp), %dx
1578 inb (%dx)
1579 ret
1580 SET_SIZE(inb)
1581
1582 #endif /* __i386 */
1583 #endif /* __lint */
1584
1585
1586 #if defined(__lint)
1587
1588 /* ARGSUSED */
1589 void
1590 repoutsw(int port, uint16_t *addr, int cnt)
1591 {}
1592
1593 #else /* __lint */
1594
1595 #if defined(__amd64)
1596
1597 ENTRY(repoutsw)
1598 movl %edx, %ecx
1599 movw %di, %dx
1600 rep
1601 D16 outsl
1602 ret
1603 SET_SIZE(repoutsw)
1604
1605 #elif defined(__i386)
1606
1607 /*
1608 * The arguments and saved registers are on the stack in the
1609 * following order:
1610 * | cnt | +16
1611 * | *addr | +12
1612 * | port | +8
1613 * | eip | +4
1614 * | esi | <-- %esp
1615 * If additional values are pushed onto the stack, make sure
1616 * to adjust the following constants accordingly.
1617 */
1618 .set PORT, 8
1619 .set ADDR, 12
1620 .set COUNT, 16
1621
1622 ENTRY(repoutsw)
1623 pushl %esi
1624 movl PORT(%esp), %edx
1625 movl ADDR(%esp), %esi
1626 movl COUNT(%esp), %ecx
1627 rep
1628 D16 outsl
1629 popl %esi
1630 ret
1631 SET_SIZE(repoutsw)
1632
1633 #endif /* __i386 */
1634 #endif /* __lint */
1635
1636
1637 #if defined(__lint)
1638
1639 /* ARGSUSED */
1640 void
1641 repinsw(int port_addr, uint16_t *addr, int cnt)
1642 {}
1643
1644 #else /* __lint */
1645
1646 #if defined(__amd64)
1647
1648 ENTRY(repinsw)
1649 movl %edx, %ecx
1650 movw %di, %dx
1651 rep
1652 D16 insl
1653 ret
1654 SET_SIZE(repinsw)
1655
1656 #elif defined(__i386)
1657
1658 ENTRY(repinsw)
1659 pushl %edi
1660 movl PORT(%esp), %edx
1661 movl ADDR(%esp), %edi
1662 movl COUNT(%esp), %ecx
1663 rep
1664 D16 insl
1665 popl %edi
1666 ret
1667 SET_SIZE(repinsw)
1668
1669 #endif /* __i386 */
1670 #endif /* __lint */
1671
1672
1673 #if defined(__lint)
1674
1675 /* ARGSUSED */
1676 void
1677 repinsb(int port, uint8_t *addr, int count)
1678 {}
1679
1680 #else /* __lint */
1681
1682 #if defined(__amd64)
1683
1684 ENTRY(repinsb)
1685 movl %edx, %ecx
1686 movw %di, %dx
1687 movq %rsi, %rdi
1688 rep
1689 insb
1690 ret
1691 SET_SIZE(repinsb)
1692
1693 #elif defined(__i386)
1694
1695 /*
1696 * The arguments and saved registers are on the stack in the
1697 * following order:
1698 * | cnt | +16
1699 * | *addr | +12
1700 * | port | +8
1701 * | eip | +4
1702 * | esi | <-- %esp
1703 * If additional values are pushed onto the stack, make sure
1704 * to adjust the following constants accordingly.
1705 */
1706 .set IO_PORT, 8
1707 .set IO_ADDR, 12
1708 .set IO_COUNT, 16
1709
1710 ENTRY(repinsb)
1711 pushl %edi
1712 movl IO_ADDR(%esp), %edi
1713 movl IO_COUNT(%esp), %ecx
1714 movl IO_PORT(%esp), %edx
1715 rep
1716 insb
1717 popl %edi
1718 ret
1719 SET_SIZE(repinsb)
1720
1721 #endif /* __i386 */
1722 #endif /* __lint */
1723
1724
1725 /*
1726 * Input a stream of 32-bit words.
1727 * NOTE: count is a DWORD count.
1728 */
1729 #if defined(__lint)
1730
1731 /* ARGSUSED */
1732 void
1733 repinsd(int port, uint32_t *addr, int count)
1734 {}
1735
1736 #else /* __lint */
1737
1738 #if defined(__amd64)
1739
1740 ENTRY(repinsd)
1741 movl %edx, %ecx
1742 movw %di, %dx
1743 movq %rsi, %rdi
1744 rep
1745 insl
1746 ret
1747 SET_SIZE(repinsd)
1748
1749 #elif defined(__i386)
1750
1751 ENTRY(repinsd)
1752 pushl %edi
1753 movl IO_ADDR(%esp), %edi
1754 movl IO_COUNT(%esp), %ecx
1755 movl IO_PORT(%esp), %edx
1756 rep
1757 insl
1758 popl %edi
1759 ret
1760 SET_SIZE(repinsd)
1761
1762 #endif /* __i386 */
1763 #endif /* __lint */
1764
1765 /*
1766 * Output a stream of bytes
1767 * NOTE: count is a byte count
1768 */
1769 #if defined(__lint)
1770
1771 /* ARGSUSED */
1772 void
1773 repoutsb(int port, uint8_t *addr, int count)
1774 {}
1775
1776 #else /* __lint */
1777
1778 #if defined(__amd64)
1779
1780 ENTRY(repoutsb)
1781 movl %edx, %ecx
1782 movw %di, %dx
1783 rep
1784 outsb
1785 ret
1786 SET_SIZE(repoutsb)
1787
1788 #elif defined(__i386)
1789
1790 ENTRY(repoutsb)
1791 pushl %esi
1792 movl IO_ADDR(%esp), %esi
1793 movl IO_COUNT(%esp), %ecx
1794 movl IO_PORT(%esp), %edx
1795 rep
1796 outsb
1797 popl %esi
1798 ret
1799 SET_SIZE(repoutsb)
1800
1801 #endif /* __i386 */
1802 #endif /* __lint */
1803
1804 /*
1805 * Output a stream of 32-bit words
1806 * NOTE: count is a DWORD count
1807 */
1808 #if defined(__lint)
1809
1810 /* ARGSUSED */
1811 void
1812 repoutsd(int port, uint32_t *addr, int count)
1813 {}
1814
1815 #else /* __lint */
1816
1817 #if defined(__amd64)
1818
1819 ENTRY(repoutsd)
1820 movl %edx, %ecx
1821 movw %di, %dx
1822 rep
1823 outsl
1824 ret
1825 SET_SIZE(repoutsd)
1826
1827 #elif defined(__i386)
1828
1829 ENTRY(repoutsd)
1830 pushl %esi
1831 movl IO_ADDR(%esp), %esi
1832 movl IO_COUNT(%esp), %ecx
1833 movl IO_PORT(%esp), %edx
1834 rep
1835 outsl
1836 popl %esi
1837 ret
1838 SET_SIZE(repoutsd)
1839
1840 #endif /* __i386 */
1841 #endif /* __lint */
1842
1843 /*
1844 * void int3(void)
1845 * void int18(void)
1846 * void int20(void)
1847 * void int_cmci(void)
1848 */
1849
1850 #if defined(__lint)
1851
1852 void
1853 int3(void)
1854 {}
1855
1856 void
1857 int18(void)
1858 {}
1859
1860 void
1861 int20(void)
1862 {}
1863
1864 void
1865 int_cmci(void)
1866 {}
1867
1868 #else /* __lint */
1869
1870 ENTRY(int3)
1871 int $T_BPTFLT
1872 ret
1873 SET_SIZE(int3)
1874
1875 ENTRY(int18)
1876 int $T_MCE
1877 ret
1878 SET_SIZE(int18)
1879
1880 ENTRY(int20)
1881 movl boothowto, %eax
1882 andl $RB_DEBUG, %eax
1883 jz 1f
1884
1885 int $T_DBGENTR
1886 1:
1887 rep; ret /* use 2 byte return instruction when branch target */
1888 /* AMD Software Optimization Guide - Section 6.2 */
1889 SET_SIZE(int20)
1890
1891 ENTRY(int_cmci)
1892 int $T_ENOEXTFLT
1893 ret
1894 SET_SIZE(int_cmci)
1895
1896 #endif /* __lint */
1897
1898 #if defined(__lint)
1899
1900 /* ARGSUSED */
1901 int
1902 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1903 { return (0); }
1904
1905 #else /* __lint */
1906
1907 #if defined(__amd64)
1908
1909 ENTRY(scanc)
1910 /* rdi == size */
1911 /* rsi == cp */
1912 /* rdx == table */
1913 /* rcx == mask */
1914 addq %rsi, %rdi /* end = &cp[size] */
1915 .scanloop:
1916 cmpq %rdi, %rsi /* while (cp < end */
1917 jnb .scandone
1918 movzbq (%rsi), %r8 /* %r8 = *cp */
1919 incq %rsi /* cp++ */
1920 testb %cl, (%r8, %rdx)
1921 jz .scanloop /* && (table[*cp] & mask) == 0) */
1922 decq %rsi /* (fix post-increment) */
1923 .scandone:
1924 movl %edi, %eax
1925 subl %esi, %eax /* return (end - cp) */
1926 ret
1927 SET_SIZE(scanc)
1928
1929 #elif defined(__i386)
1930
1931 ENTRY(scanc)
1932 pushl %edi
1933 pushl %esi
1934 movb 24(%esp), %cl /* mask = %cl */
1935 movl 16(%esp), %esi /* cp = %esi */
1936 movl 20(%esp), %edx /* table = %edx */
1937 movl %esi, %edi
1938 addl 12(%esp), %edi /* end = &cp[size]; */
1939 .scanloop:
1940 cmpl %edi, %esi /* while (cp < end */
1941 jnb .scandone
1942 movzbl (%esi), %eax /* %al = *cp */
1943 incl %esi /* cp++ */
1944 movb (%edx, %eax), %al /* %al = table[*cp] */
1945 testb %al, %cl
1946 jz .scanloop /* && (table[*cp] & mask) == 0) */
1947 dec %esi /* post-incremented */
1948 .scandone:
1949 movl %edi, %eax
1950 subl %esi, %eax /* return (end - cp) */
1951 popl %esi
1952 popl %edi
1953 ret
1954 SET_SIZE(scanc)
1955
1956 #endif /* __i386 */
1957 #endif /* __lint */
1958
1959 /*
1960 * Replacement functions for ones that are normally inlined.
1961 * In addition to the copy in i86.il, they are defined here just in case.
1962 */
1963
1964 #if defined(__lint)
1965
1966 ulong_t
1967 intr_clear(void)
1968 { return (0); }
1969
1970 ulong_t
1971 clear_int_flag(void)
1972 { return (0); }
1973
1974 #else /* __lint */
1975
1976 #if defined(__amd64)
1977
1978 ENTRY(intr_clear)
1979 ENTRY(clear_int_flag)
1980 pushfq
1981 popq %rax
1982 #if defined(__xpv)
1983 leaq xpv_panicking, %rdi
1984 movl (%rdi), %edi
1985 cmpl $0, %edi
1986 jne 2f
1987 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1988 /*
1989 * Synthesize the PS_IE bit from the event mask bit
1990 */
1991 andq $_BITNOT(PS_IE), %rax
1992 testb $1, %dl
1993 jnz 1f
1994 orq $PS_IE, %rax
1995 1:
1996 ret
1997 2:
1998 #endif
1999 CLI(%rdi)
2000 ret
2001 SET_SIZE(clear_int_flag)
2002 SET_SIZE(intr_clear)
2003
2004 #elif defined(__i386)
2005
2006 ENTRY(intr_clear)
2007 ENTRY(clear_int_flag)
2008 pushfl
2009 popl %eax
2010 #if defined(__xpv)
2011 leal xpv_panicking, %edx
2012 movl (%edx), %edx
2013 cmpl $0, %edx
2014 jne 2f
2015 CLIRET(%edx, %cl) /* returns event mask in %cl */
2016 /*
2017 * Synthesize the PS_IE bit from the event mask bit
2018 */
2019 andl $_BITNOT(PS_IE), %eax
2020 testb $1, %cl
2021 jnz 1f
2022 orl $PS_IE, %eax
2023 1:
2024 ret
2025 2:
2026 #endif
2027 CLI(%edx)
2028 ret
2029 SET_SIZE(clear_int_flag)
2030 SET_SIZE(intr_clear)
2031
2032 #endif /* __i386 */
2033 #endif /* __lint */
2034
2035 #if defined(__lint)
2036
2037 struct cpu *
2038 curcpup(void)
2039 { return 0; }
2040
2041 #else /* __lint */
2042
2043 #if defined(__amd64)
2044
2045 ENTRY(curcpup)
2046 movq %gs:CPU_SELF, %rax
2047 ret
2048 SET_SIZE(curcpup)
2049
2050 #elif defined(__i386)
2051
2052 ENTRY(curcpup)
2053 movl %gs:CPU_SELF, %eax
2054 ret
2055 SET_SIZE(curcpup)
2056
2057 #endif /* __i386 */
2058 #endif /* __lint */
2059
2060 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2061 * These functions reverse the byte order of the input parameter and returns
2062 * the result. This is to convert the byte order from host byte order
2063 * (little endian) to network byte order (big endian), or vice versa.
2064 */
2065
2066 #if defined(__lint)
2067
2068 uint64_t
2069 htonll(uint64_t i)
2070 { return (i); }
2071
2072 uint64_t
2073 ntohll(uint64_t i)
2074 { return (i); }
2075
2076 uint32_t
2077 htonl(uint32_t i)
2078 { return (i); }
2079
2080 uint32_t
2081 ntohl(uint32_t i)
2082 { return (i); }
2083
2084 uint16_t
2085 htons(uint16_t i)
2086 { return (i); }
2087
2088 uint16_t
2089 ntohs(uint16_t i)
2090 { return (i); }
2091
2092 #else /* __lint */
2093
2094 #if defined(__amd64)
2095
2096 ENTRY(htonll)
2097 ALTENTRY(ntohll)
2098 movq %rdi, %rax
2099 bswapq %rax
2100 ret
2101 SET_SIZE(ntohll)
2102 SET_SIZE(htonll)
2103
2104 /* XX64 there must be shorter sequences for this */
2105 ENTRY(htonl)
2106 ALTENTRY(ntohl)
2107 movl %edi, %eax
2108 bswap %eax
2109 ret
2110 SET_SIZE(ntohl)
2111 SET_SIZE(htonl)
2112
2113 /* XX64 there must be better sequences for this */
2114 ENTRY(htons)
2115 ALTENTRY(ntohs)
2116 movl %edi, %eax
2117 bswap %eax
2118 shrl $16, %eax
2119 ret
2120 SET_SIZE(ntohs)
2121 SET_SIZE(htons)
2122
2123 #elif defined(__i386)
2124
2125 ENTRY(htonll)
2126 ALTENTRY(ntohll)
2127 movl 4(%esp), %edx
2128 movl 8(%esp), %eax
2129 bswap %edx
2130 bswap %eax
2131 ret
2132 SET_SIZE(ntohll)
2133 SET_SIZE(htonll)
2134
2135 ENTRY(htonl)
2136 ALTENTRY(ntohl)
2137 movl 4(%esp), %eax
2138 bswap %eax
2139 ret
2140 SET_SIZE(ntohl)
2141 SET_SIZE(htonl)
2142
2143 ENTRY(htons)
2144 ALTENTRY(ntohs)
2145 movl 4(%esp), %eax
2146 bswap %eax
2147 shrl $16, %eax
2148 ret
2149 SET_SIZE(ntohs)
2150 SET_SIZE(htons)
2151
2152 #endif /* __i386 */
2153 #endif /* __lint */
2154
2155
2156 #if defined(__lint)
2157
2158 /* ARGSUSED */
2159 void
2160 intr_restore(ulong_t i)
2161 { return; }
2162
2163 /* ARGSUSED */
2164 void
2165 restore_int_flag(ulong_t i)
2166 { return; }
2167
2168 #else /* __lint */
2169
2170 #if defined(__amd64)
2171
2172 ENTRY(intr_restore)
2173 ENTRY(restore_int_flag)
2174 testq $PS_IE, %rdi
2175 jz 1f
2176 #if defined(__xpv)
2177 leaq xpv_panicking, %rsi
2178 movl (%rsi), %esi
2179 cmpl $0, %esi
2180 jne 1f
2181 /*
2182 * Since we're -really- running unprivileged, our attempt
2183 * to change the state of the IF bit will be ignored.
2184 * The virtual IF bit is tweaked by CLI and STI.
2185 */
2186 IE_TO_EVENT_MASK(%rsi, %rdi)
2187 #else
2188 sti
2189 #endif
2190 1:
2191 ret
2192 SET_SIZE(restore_int_flag)
2193 SET_SIZE(intr_restore)
2194
2195 #elif defined(__i386)
2196
2197 ENTRY(intr_restore)
2198 ENTRY(restore_int_flag)
2199 testl $PS_IE, 4(%esp)
2200 jz 1f
2201 #if defined(__xpv)
2202 leal xpv_panicking, %edx
2203 movl (%edx), %edx
2204 cmpl $0, %edx
2205 jne 1f
2206 /*
2207 * Since we're -really- running unprivileged, our attempt
2208 * to change the state of the IF bit will be ignored.
2209 * The virtual IF bit is tweaked by CLI and STI.
2210 */
2211 IE_TO_EVENT_MASK(%edx, 4(%esp))
2212 #else
2213 sti
2214 #endif
2215 1:
2216 ret
2217 SET_SIZE(restore_int_flag)
2218 SET_SIZE(intr_restore)
2219
2220 #endif /* __i386 */
2221 #endif /* __lint */
2222
2223 #if defined(__lint)
2224
2225 void
2226 sti(void)
2227 {}
2228
2229 void
2230 cli(void)
2231 {}
2232
2233 #else /* __lint */
2234
2235 ENTRY(sti)
2236 STI
2237 ret
2238 SET_SIZE(sti)
2239
2240 ENTRY(cli)
2241 #if defined(__amd64)
2242 CLI(%rax)
2243 #elif defined(__i386)
2244 CLI(%eax)
2245 #endif /* __i386 */
2246 ret
2247 SET_SIZE(cli)
2248
2249 #endif /* __lint */
2250
2251 #if defined(__lint)
2252
2253 dtrace_icookie_t
2254 dtrace_interrupt_disable(void)
2255 { return (0); }
2256
2257 #else /* __lint */
2258
2259 #if defined(__amd64)
2260
2261 ENTRY(dtrace_interrupt_disable)
2262 pushfq
2263 popq %rax
2264 #if defined(__xpv)
2265 leaq xpv_panicking, %rdi
2266 movl (%rdi), %edi
2267 cmpl $0, %edi
2268 jne .dtrace_interrupt_disable_done
2269 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2270 /*
2271 * Synthesize the PS_IE bit from the event mask bit
2272 */
2273 andq $_BITNOT(PS_IE), %rax
2274 testb $1, %dl
2275 jnz .dtrace_interrupt_disable_done
2276 orq $PS_IE, %rax
2277 #else
2278 CLI(%rdx)
2279 #endif
2280 .dtrace_interrupt_disable_done:
2281 ret
2282 SET_SIZE(dtrace_interrupt_disable)
2283
2284 #elif defined(__i386)
2285
2286 ENTRY(dtrace_interrupt_disable)
2287 pushfl
2288 popl %eax
2289 #if defined(__xpv)
2290 leal xpv_panicking, %edx
2291 movl (%edx), %edx
2292 cmpl $0, %edx
2293 jne .dtrace_interrupt_disable_done
2294 CLIRET(%edx, %cl) /* returns event mask in %cl */
2295 /*
2296 * Synthesize the PS_IE bit from the event mask bit
2297 */
2298 andl $_BITNOT(PS_IE), %eax
2299 testb $1, %cl
2300 jnz .dtrace_interrupt_disable_done
2301 orl $PS_IE, %eax
2302 #else
2303 CLI(%edx)
2304 #endif
2305 .dtrace_interrupt_disable_done:
2306 ret
2307 SET_SIZE(dtrace_interrupt_disable)
2308
2309 #endif /* __i386 */
2310 #endif /* __lint */
2311
2312 #if defined(__lint)
2313
2314 /*ARGSUSED*/
2315 void
2316 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2317 {}
2318
2319 #else /* __lint */
2320
2321 #if defined(__amd64)
2322
2323 ENTRY(dtrace_interrupt_enable)
2324 pushq %rdi
2325 popfq
2326 #if defined(__xpv)
2327 leaq xpv_panicking, %rdx
2328 movl (%rdx), %edx
2329 cmpl $0, %edx
2330 jne .dtrace_interrupt_enable_done
2331 /*
2332 * Since we're -really- running unprivileged, our attempt
2333 * to change the state of the IF bit will be ignored. The
2334 * virtual IF bit is tweaked by CLI and STI.
2335 */
2336 IE_TO_EVENT_MASK(%rdx, %rdi)
2337 #endif
2338 .dtrace_interrupt_enable_done:
2339 ret
2340 SET_SIZE(dtrace_interrupt_enable)
2341
2342 #elif defined(__i386)
2343
2344 ENTRY(dtrace_interrupt_enable)
2345 movl 4(%esp), %eax
2346 pushl %eax
2347 popfl
2348 #if defined(__xpv)
2349 leal xpv_panicking, %edx
2350 movl (%edx), %edx
2351 cmpl $0, %edx
2352 jne .dtrace_interrupt_enable_done
2353 /*
2354 * Since we're -really- running unprivileged, our attempt
2355 * to change the state of the IF bit will be ignored. The
2356 * virtual IF bit is tweaked by CLI and STI.
2357 */
2358 IE_TO_EVENT_MASK(%edx, %eax)
2359 #endif
2360 .dtrace_interrupt_enable_done:
2361 ret
2362 SET_SIZE(dtrace_interrupt_enable)
2363
2364 #endif /* __i386 */
2365 #endif /* __lint */
2366
2367
2368 #if defined(lint)
2369
2370 void
2371 dtrace_membar_producer(void)
2372 {}
2373
2374 void
2375 dtrace_membar_consumer(void)
2376 {}
2377
2378 #else /* __lint */
2379
2380 ENTRY(dtrace_membar_producer)
2381 rep; ret /* use 2 byte return instruction when branch target */
2382 /* AMD Software Optimization Guide - Section 6.2 */
2383 SET_SIZE(dtrace_membar_producer)
2384
2385 ENTRY(dtrace_membar_consumer)
2386 rep; ret /* use 2 byte return instruction when branch target */
2387 /* AMD Software Optimization Guide - Section 6.2 */
2388 SET_SIZE(dtrace_membar_consumer)
2389
2390 #endif /* __lint */
2391
2392 #if defined(__lint)
2393
2394 kthread_id_t
2395 threadp(void)
2396 { return ((kthread_id_t)0); }
2397
2398 #else /* __lint */
2399
2400 #if defined(__amd64)
2401
2402 ENTRY(threadp)
2403 movq %gs:CPU_THREAD, %rax
2404 ret
2405 SET_SIZE(threadp)
2406
2407 #elif defined(__i386)
2408
2409 ENTRY(threadp)
2410 movl %gs:CPU_THREAD, %eax
2411 ret
2412 SET_SIZE(threadp)
2413
2414 #endif /* __i386 */
2415 #endif /* __lint */
2416
2417 /*
2418 * Checksum routine for Internet Protocol Headers
2419 */
2420
2421 #if defined(__lint)
2422
2423 /* ARGSUSED */
2424 unsigned int
2425 ip_ocsum(
2426 ushort_t *address, /* ptr to 1st message buffer */
2427 int halfword_count, /* length of data */
2428 unsigned int sum) /* partial checksum */
2429 {
2430 int i;
2431 unsigned int psum = 0; /* partial sum */
2432
2433 for (i = 0; i < halfword_count; i++, address++) {
2434 psum += *address;
2435 }
2436
2437 while ((psum >> 16) != 0) {
2438 psum = (psum & 0xffff) + (psum >> 16);
2439 }
2440
2441 psum += sum;
2442
2443 while ((psum >> 16) != 0) {
2444 psum = (psum & 0xffff) + (psum >> 16);
2445 }
2446
2447 return (psum);
2448 }
2449
2450 #else /* __lint */
2451
2452 #if defined(__amd64)
2453
2454 ENTRY(ip_ocsum)
2455 pushq %rbp
2456 movq %rsp, %rbp
2457 #ifdef DEBUG
2458 movq postbootkernelbase(%rip), %rax
2459 cmpq %rax, %rdi
2460 jnb 1f
2461 xorl %eax, %eax
2462 movq %rdi, %rsi
2463 leaq .ip_ocsum_panic_msg(%rip), %rdi
2464 call panic
2465 /*NOTREACHED*/
2466 .ip_ocsum_panic_msg:
2467 .string "ip_ocsum: address 0x%p below kernelbase\n"
2468 1:
2469 #endif
2470 movl %esi, %ecx /* halfword_count */
2471 movq %rdi, %rsi /* address */
2472 /* partial sum in %edx */
2473 xorl %eax, %eax
2474 testl %ecx, %ecx
2475 jz .ip_ocsum_done
2476 testq $3, %rsi
2477 jnz .ip_csum_notaligned
2478 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2479 .next_iter:
2480 /* XX64 opportunities for prefetch? */
2481 /* XX64 compute csum with 64 bit quantities? */
2482 subl $32, %ecx
2483 jl .less_than_32
2484
2485 addl 0(%rsi), %edx
2486 .only60:
2487 adcl 4(%rsi), %eax
2488 .only56:
2489 adcl 8(%rsi), %edx
2490 .only52:
2491 adcl 12(%rsi), %eax
2492 .only48:
2493 adcl 16(%rsi), %edx
2494 .only44:
2495 adcl 20(%rsi), %eax
2496 .only40:
2497 adcl 24(%rsi), %edx
2498 .only36:
2499 adcl 28(%rsi), %eax
2500 .only32:
2501 adcl 32(%rsi), %edx
2502 .only28:
2503 adcl 36(%rsi), %eax
2504 .only24:
2505 adcl 40(%rsi), %edx
2506 .only20:
2507 adcl 44(%rsi), %eax
2508 .only16:
2509 adcl 48(%rsi), %edx
2510 .only12:
2511 adcl 52(%rsi), %eax
2512 .only8:
2513 adcl 56(%rsi), %edx
2514 .only4:
2515 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2516 .only0:
2517 adcl $0, %eax /* could be adding -1 in eax with a carry */
2518 adcl $0, %eax
2519
2520 addq $64, %rsi
2521 testl %ecx, %ecx
2522 jnz .next_iter
2523
2524 .ip_ocsum_done:
2525 addl %eax, %edx
2526 adcl $0, %edx
2527 movl %edx, %eax /* form a 16 bit checksum by */
2528 shrl $16, %eax /* adding two halves of 32 bit checksum */
2529 addw %dx, %ax
2530 adcw $0, %ax
2531 andl $0xffff, %eax
2532 leave
2533 ret
2534
2535 .ip_csum_notaligned:
2536 xorl %edi, %edi
2537 movw (%rsi), %di
2538 addl %edi, %edx
2539 adcl $0, %edx
2540 addq $2, %rsi
2541 decl %ecx
2542 jmp .ip_csum_aligned
2543
2544 .less_than_32:
2545 addl $32, %ecx
2546 testl $1, %ecx
2547 jz .size_aligned
2548 andl $0xfe, %ecx
2549 movzwl (%rsi, %rcx, 2), %edi
2550 addl %edi, %edx
2551 adcl $0, %edx
2552 .size_aligned:
2553 movl %ecx, %edi
2554 shrl $1, %ecx
2555 shl $1, %edi
2556 subq $64, %rdi
2557 addq %rdi, %rsi
2558 leaq .ip_ocsum_jmptbl(%rip), %rdi
2559 leaq (%rdi, %rcx, 8), %rdi
2560 xorl %ecx, %ecx
2561 clc
2562 jmp *(%rdi)
2563
2564 .align 8
2565 .ip_ocsum_jmptbl:
2566 .quad .only0, .only4, .only8, .only12, .only16, .only20
2567 .quad .only24, .only28, .only32, .only36, .only40, .only44
2568 .quad .only48, .only52, .only56, .only60
2569 SET_SIZE(ip_ocsum)
2570
2571 #elif defined(__i386)
2572
2573 ENTRY(ip_ocsum)
2574 pushl %ebp
2575 movl %esp, %ebp
2576 pushl %ebx
2577 pushl %esi
2578 pushl %edi
2579 movl 12(%ebp), %ecx /* count of half words */
2580 movl 16(%ebp), %edx /* partial checksum */
2581 movl 8(%ebp), %esi
2582 xorl %eax, %eax
2583 testl %ecx, %ecx
2584 jz .ip_ocsum_done
2585
2586 testl $3, %esi
2587 jnz .ip_csum_notaligned
2588 .ip_csum_aligned:
2589 .next_iter:
2590 subl $32, %ecx
2591 jl .less_than_32
2592
2593 addl 0(%esi), %edx
2594 .only60:
2595 adcl 4(%esi), %eax
2596 .only56:
2597 adcl 8(%esi), %edx
2598 .only52:
2599 adcl 12(%esi), %eax
2600 .only48:
2601 adcl 16(%esi), %edx
2602 .only44:
2603 adcl 20(%esi), %eax
2604 .only40:
2605 adcl 24(%esi), %edx
2606 .only36:
2607 adcl 28(%esi), %eax
2608 .only32:
2609 adcl 32(%esi), %edx
2610 .only28:
2611 adcl 36(%esi), %eax
2612 .only24:
2613 adcl 40(%esi), %edx
2614 .only20:
2615 adcl 44(%esi), %eax
2616 .only16:
2617 adcl 48(%esi), %edx
2618 .only12:
2619 adcl 52(%esi), %eax
2620 .only8:
2621 adcl 56(%esi), %edx
2622 .only4:
2623 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2624 .only0:
2625 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2626 adcl $0, %eax
2627
2628 addl $64, %esi
2629 andl %ecx, %ecx
2630 jnz .next_iter
2631
2632 .ip_ocsum_done:
2633 addl %eax, %edx
2634 adcl $0, %edx
2635 movl %edx, %eax /* form a 16 bit checksum by */
2636 shrl $16, %eax /* adding two halves of 32 bit checksum */
2637 addw %dx, %ax
2638 adcw $0, %ax
2639 andl $0xffff, %eax
2640 popl %edi /* restore registers */
2641 popl %esi
2642 popl %ebx
2643 leave
2644 ret
2645
2646 .ip_csum_notaligned:
2647 xorl %edi, %edi
2648 movw (%esi), %di
2649 addl %edi, %edx
2650 adcl $0, %edx
2651 addl $2, %esi
2652 decl %ecx
2653 jmp .ip_csum_aligned
2654
2655 .less_than_32:
2656 addl $32, %ecx
2657 testl $1, %ecx
2658 jz .size_aligned
2659 andl $0xfe, %ecx
2660 movzwl (%esi, %ecx, 2), %edi
2661 addl %edi, %edx
2662 adcl $0, %edx
2663 .size_aligned:
2664 movl %ecx, %edi
2665 shrl $1, %ecx
2666 shl $1, %edi
2667 subl $64, %edi
2668 addl %edi, %esi
2669 movl $.ip_ocsum_jmptbl, %edi
2670 lea (%edi, %ecx, 4), %edi
2671 xorl %ecx, %ecx
2672 clc
2673 jmp *(%edi)
2674 SET_SIZE(ip_ocsum)
2675
2676 .data
2677 .align 4
2678
2679 .ip_ocsum_jmptbl:
2680 .long .only0, .only4, .only8, .only12, .only16, .only20
2681 .long .only24, .only28, .only32, .only36, .only40, .only44
2682 .long .only48, .only52, .only56, .only60
2683
2684
2685 #endif /* __i386 */
2686 #endif /* __lint */
2687
2688 /*
2689 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2690 * Provided to manipulate hrtime_t values.
2691 */
2692 #if defined(__lint)
2693
2694 /* result = a * b; */
2695
2696 /* ARGSUSED */
2697 unsigned long long
2698 mul32(uint_t a, uint_t b)
2699 { return (0); }
2700
2701 #else /* __lint */
2702
2703 #if defined(__amd64)
2704
2705 ENTRY(mul32)
2706 xorl %edx, %edx /* XX64 joe, paranoia? */
2707 movl %edi, %eax
2708 mull %esi
2709 shlq $32, %rdx
2710 orq %rdx, %rax
2711 ret
2712 SET_SIZE(mul32)
2713
2714 #elif defined(__i386)
2715
2716 ENTRY(mul32)
2717 movl 8(%esp), %eax
2718 movl 4(%esp), %ecx
2719 mull %ecx
2720 ret
2721 SET_SIZE(mul32)
2722
2723 #endif /* __i386 */
2724 #endif /* __lint */
2725
2726 #if defined(notused)
2727 #if defined(__lint)
2728 /* ARGSUSED */
2729 void
2730 load_pte64(uint64_t *pte, uint64_t pte_value)
2731 {}
2732 #else /* __lint */
2733 .globl load_pte64
2734 load_pte64:
2735 movl 4(%esp), %eax
2736 movl 8(%esp), %ecx
2737 movl 12(%esp), %edx
2738 movl %edx, 4(%eax)
2739 movl %ecx, (%eax)
2740 ret
2741 #endif /* __lint */
2742 #endif /* notused */
2743
2744 #if defined(__lint)
2745
2746 /*ARGSUSED*/
2747 void
2748 scan_memory(caddr_t addr, size_t size)
2749 {}
2750
2751 #else /* __lint */
2752
2753 #if defined(__amd64)
2754
2755 ENTRY(scan_memory)
2756 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2757 jz .scanm_done
2758 movq %rsi, %rcx /* move count into rep control register */
2759 movq %rdi, %rsi /* move addr into lodsq control reg. */
2760 rep lodsq /* scan the memory range */
2761 .scanm_done:
2762 rep; ret /* use 2 byte return instruction when branch target */
2763 /* AMD Software Optimization Guide - Section 6.2 */
2764 SET_SIZE(scan_memory)
2765
2766 #elif defined(__i386)
2767
2768 ENTRY(scan_memory)
2769 pushl %ecx
2770 pushl %esi
2771 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2772 shrl $2, %ecx /* convert from byte count to word count */
2773 jz .scanm_done
2774 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2775 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2776 lodsl
2777 .scanm_done:
2778 popl %esi
2779 popl %ecx
2780 ret
2781 SET_SIZE(scan_memory)
2782
2783 #endif /* __i386 */
2784 #endif /* __lint */
2785
2786
2787 #if defined(__lint)
2788
2789 /*ARGSUSED */
2790 int
2791 lowbit(ulong_t i)
2792 { return (0); }
2793
2794 #else /* __lint */
2795
2796 #if defined(__amd64)
2797
2798 ENTRY(lowbit)
2799 movl $-1, %eax
2800 bsfq %rdi, %rdi
2801 cmovnz %edi, %eax
2802 incl %eax
2803 ret
2804 SET_SIZE(lowbit)
2805
2806 #elif defined(__i386)
2807
2808 ENTRY(lowbit)
2809 bsfl 4(%esp), %eax
2810 jz 0f
2811 incl %eax
2812 ret
2813 0:
2814 xorl %eax, %eax
2815 ret
2816 SET_SIZE(lowbit)
2817
2818 #endif /* __i386 */
2819 #endif /* __lint */
2820
2821 #if defined(__lint)
2822
2823 /*ARGSUSED*/
2824 int
2825 highbit(ulong_t i)
2826 { return (0); }
2827
2828 /*ARGSUSED*/
2829 int
2830 highbit64(uint64_t i)
2831 { return (0); }
2832
2833 #else /* __lint */
2834
2835 #if defined(__amd64)
2836
2837 ENTRY(highbit)
2838 ALTENTRY(highbit64)
2839 movl $-1, %eax
2840 bsrq %rdi, %rdi
2841 cmovnz %edi, %eax
2842 incl %eax
2843 ret
2844 SET_SIZE(highbit64)
2845 SET_SIZE(highbit)
2846
2847 #elif defined(__i386)
2848
2849 ENTRY(highbit)
2850 bsrl 4(%esp), %eax
2851 jz 0f
2852 incl %eax
2853 ret
2854 0:
2855 xorl %eax, %eax
2856 ret
2857 SET_SIZE(highbit)
2858
2859 ENTRY(highbit64)
2860 bsrl 8(%esp), %eax
2861 jz highbit
2862 addl $33, %eax
2863 ret
2864 SET_SIZE(highbit64)
2865
2866 #endif /* __i386 */
2867 #endif /* __lint */
2868
2869 #if defined(__lint)
2870
2871 /*ARGSUSED*/
2872 uint64_t
2873 rdmsr(uint_t r)
2874 { return (0); }
2875
2876 /*ARGSUSED*/
2877 void
2878 wrmsr(uint_t r, const uint64_t val)
2879 {}
2880
2881 /*ARGSUSED*/
2882 uint64_t
2883 xrdmsr(uint_t r)
2884 { return (0); }
2885
2886 /*ARGSUSED*/
2887 void
2888 xwrmsr(uint_t r, const uint64_t val)
2889 {}
2890
2891 void
2892 invalidate_cache(void)
2893 {}
2894
2895 /*ARGSUSED*/
2896 uint64_t
2897 get_xcr(uint_t r)
2898 { return (0); }
2899
2900 /*ARGSUSED*/
2901 void
2902 set_xcr(uint_t r, const uint64_t val)
2903 {}
2904
2905 #else /* __lint */
2906
2907 #define XMSR_ACCESS_VAL $0x9c5a203a
2908
2909 #if defined(__amd64)
2910
2911 ENTRY(rdmsr)
2912 movl %edi, %ecx
2913 rdmsr
2914 shlq $32, %rdx
2915 orq %rdx, %rax
2916 ret
2917 SET_SIZE(rdmsr)
2918
2919 ENTRY(wrmsr)
2920 movq %rsi, %rdx
2921 shrq $32, %rdx
2922 movl %esi, %eax
2923 movl %edi, %ecx
2924 wrmsr
2925 ret
2926 SET_SIZE(wrmsr)
2927
2928 ENTRY(xrdmsr)
2929 pushq %rbp
2930 movq %rsp, %rbp
2931 movl %edi, %ecx
2932 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2933 rdmsr
2934 shlq $32, %rdx
2935 orq %rdx, %rax
2936 leave
2937 ret
2938 SET_SIZE(xrdmsr)
2939
2940 ENTRY(xwrmsr)
2941 pushq %rbp
2942 movq %rsp, %rbp
2943 movl %edi, %ecx
2944 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2945 movq %rsi, %rdx
2946 shrq $32, %rdx
2947 movl %esi, %eax
2948 wrmsr
2949 leave
2950 ret
2951 SET_SIZE(xwrmsr)
2952
2953 ENTRY(get_xcr)
2954 movl %edi, %ecx
2955 #xgetbv
2956 .byte 0x0f,0x01,0xd0
2957 shlq $32, %rdx
2958 orq %rdx, %rax
2959 ret
2960 SET_SIZE(get_xcr)
2961
2962 ENTRY(set_xcr)
2963 movq %rsi, %rdx
2964 shrq $32, %rdx
2965 movl %esi, %eax
2966 movl %edi, %ecx
2967 #xsetbv
2968 .byte 0x0f,0x01,0xd1
2969 ret
2970 SET_SIZE(set_xcr)
2971
2972 #elif defined(__i386)
2973
2974 ENTRY(rdmsr)
2975 movl 4(%esp), %ecx
2976 rdmsr
2977 ret
2978 SET_SIZE(rdmsr)
2979
2980 ENTRY(wrmsr)
2981 movl 4(%esp), %ecx
2982 movl 8(%esp), %eax
2983 movl 12(%esp), %edx
2984 wrmsr
2985 ret
2986 SET_SIZE(wrmsr)
2987
2988 ENTRY(xrdmsr)
2989 pushl %ebp
2990 movl %esp, %ebp
2991 movl 8(%esp), %ecx
2992 pushl %edi
2993 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2994 rdmsr
2995 popl %edi
2996 leave
2997 ret
2998 SET_SIZE(xrdmsr)
2999
3000 ENTRY(xwrmsr)
3001 pushl %ebp
3002 movl %esp, %ebp
3003 movl 8(%esp), %ecx
3004 movl 12(%esp), %eax
3005 movl 16(%esp), %edx
3006 pushl %edi
3007 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3008 wrmsr
3009 popl %edi
3010 leave
3011 ret
3012 SET_SIZE(xwrmsr)
3013
3014 ENTRY(get_xcr)
3015 movl 4(%esp), %ecx
3016 #xgetbv
3017 .byte 0x0f,0x01,0xd0
3018 ret
3019 SET_SIZE(get_xcr)
3020
3021 ENTRY(set_xcr)
3022 movl 4(%esp), %ecx
3023 movl 8(%esp), %eax
3024 movl 12(%esp), %edx
3025 #xsetbv
3026 .byte 0x0f,0x01,0xd1
3027 ret
3028 SET_SIZE(set_xcr)
3029
3030 #endif /* __i386 */
3031
3032 ENTRY(invalidate_cache)
3033 wbinvd
3034 ret
3035 SET_SIZE(invalidate_cache)
3036
3037 #endif /* __lint */
3038
3039 #if defined(__lint)
3040
3041 /*ARGSUSED*/
3042 void
3043 getcregs(struct cregs *crp)
3044 {}
3045
3046 #else /* __lint */
3047
3048 #if defined(__amd64)
3049
3050 ENTRY_NP(getcregs)
3051 #if defined(__xpv)
3052 /*
3053 * Only a few of the hardware control registers or descriptor tables
3054 * are directly accessible to us, so just zero the structure.
3055 *
3056 * XXPV Perhaps it would be helpful for the hypervisor to return
3057 * virtualized versions of these for post-mortem use.
3058 * (Need to reevaluate - perhaps it already does!)
3059 */
3060 pushq %rdi /* save *crp */
3061 movq $CREGSZ, %rsi
3062 call bzero
3063 popq %rdi
3064
3065 /*
3066 * Dump what limited information we can
3067 */
3068 movq %cr0, %rax
3069 movq %rax, CREG_CR0(%rdi) /* cr0 */
3070 movq %cr2, %rax
3071 movq %rax, CREG_CR2(%rdi) /* cr2 */
3072 movq %cr3, %rax
3073 movq %rax, CREG_CR3(%rdi) /* cr3 */
3074 movq %cr4, %rax
3075 movq %rax, CREG_CR4(%rdi) /* cr4 */
3076
3077 #else /* __xpv */
3078
3079 #define GETMSR(r, off, d) \
3080 movl $r, %ecx; \
3081 rdmsr; \
3082 movl %eax, off(d); \
3083 movl %edx, off+4(d)
3084
3085 xorl %eax, %eax
3086 movq %rax, CREG_GDT+8(%rdi)
3087 sgdt CREG_GDT(%rdi) /* 10 bytes */
3088 movq %rax, CREG_IDT+8(%rdi)
3089 sidt CREG_IDT(%rdi) /* 10 bytes */
3090 movq %rax, CREG_LDT(%rdi)
3091 sldt CREG_LDT(%rdi) /* 2 bytes */
3092 movq %rax, CREG_TASKR(%rdi)
3093 str CREG_TASKR(%rdi) /* 2 bytes */
3094 movq %cr0, %rax
3095 movq %rax, CREG_CR0(%rdi) /* cr0 */
3096 movq %cr2, %rax
3097 movq %rax, CREG_CR2(%rdi) /* cr2 */
3098 movq %cr3, %rax
3099 movq %rax, CREG_CR3(%rdi) /* cr3 */
3100 movq %cr4, %rax
3101 movq %rax, CREG_CR4(%rdi) /* cr4 */
3102 movq %cr8, %rax
3103 movq %rax, CREG_CR8(%rdi) /* cr8 */
3104 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3105 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3106 #endif /* __xpv */
3107 ret
3108 SET_SIZE(getcregs)
3109
3110 #undef GETMSR
3111
3112 #elif defined(__i386)
3113
3114 ENTRY_NP(getcregs)
3115 #if defined(__xpv)
3116 /*
3117 * Only a few of the hardware control registers or descriptor tables
3118 * are directly accessible to us, so just zero the structure.
3119 *
3120 * XXPV Perhaps it would be helpful for the hypervisor to return
3121 * virtualized versions of these for post-mortem use.
3122 * (Need to reevaluate - perhaps it already does!)
3123 */
3124 movl 4(%esp), %edx
3125 pushl $CREGSZ
3126 pushl %edx
3127 call bzero
3128 addl $8, %esp
3129 movl 4(%esp), %edx
3130
3131 /*
3132 * Dump what limited information we can
3133 */
3134 movl %cr0, %eax
3135 movl %eax, CREG_CR0(%edx) /* cr0 */
3136 movl %cr2, %eax
3137 movl %eax, CREG_CR2(%edx) /* cr2 */
3138 movl %cr3, %eax
3139 movl %eax, CREG_CR3(%edx) /* cr3 */
3140 movl %cr4, %eax
3141 movl %eax, CREG_CR4(%edx) /* cr4 */
3142
3143 #else /* __xpv */
3144
3145 movl 4(%esp), %edx
3146 movw $0, CREG_GDT+6(%edx)
3147 movw $0, CREG_IDT+6(%edx)
3148 sgdt CREG_GDT(%edx) /* gdt */
3149 sidt CREG_IDT(%edx) /* idt */
3150 sldt CREG_LDT(%edx) /* ldt */
3151 str CREG_TASKR(%edx) /* task */
3152 movl %cr0, %eax
3153 movl %eax, CREG_CR0(%edx) /* cr0 */
3154 movl %cr2, %eax
3155 movl %eax, CREG_CR2(%edx) /* cr2 */
3156 movl %cr3, %eax
3157 movl %eax, CREG_CR3(%edx) /* cr3 */
3158 bt $X86FSET_LARGEPAGE, x86_featureset
3159 jnc .nocr4
3160 movl %cr4, %eax
3161 movl %eax, CREG_CR4(%edx) /* cr4 */
3162 jmp .skip
3163 .nocr4:
3164 movl $0, CREG_CR4(%edx)
3165 .skip:
3166 #endif
3167 ret
3168 SET_SIZE(getcregs)
3169
3170 #endif /* __i386 */
3171 #endif /* __lint */
3172
3173
3174 /*
3175 * A panic trigger is a word which is updated atomically and can only be set
3176 * once. We atomically store 0xDEFACEDD and load the old value. If the
3177 * previous value was 0, we succeed and return 1; otherwise return 0.
3178 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3179 * has its own version of this function to allow it to panic correctly from
3180 * probe context.
3181 */
3182 #if defined(__lint)
3183
3184 /*ARGSUSED*/
3185 int
3186 panic_trigger(int *tp)
3187 { return (0); }
3188
3189 /*ARGSUSED*/
3190 int
3191 dtrace_panic_trigger(int *tp)
3192 { return (0); }
3193
3194 #else /* __lint */
3195
3196 #if defined(__amd64)
3197
3198 ENTRY_NP(panic_trigger)
3199 xorl %eax, %eax
3200 movl $0xdefacedd, %edx
3201 lock
3202 xchgl %edx, (%rdi)
3203 cmpl $0, %edx
3204 je 0f
3205 movl $0, %eax
3206 ret
3207 0: movl $1, %eax
3208 ret
3209 SET_SIZE(panic_trigger)
3210
3211 ENTRY_NP(dtrace_panic_trigger)
3212 xorl %eax, %eax
3213 movl $0xdefacedd, %edx
3214 lock
3215 xchgl %edx, (%rdi)
3216 cmpl $0, %edx
3217 je 0f
3218 movl $0, %eax
3219 ret
3220 0: movl $1, %eax
3221 ret
3222 SET_SIZE(dtrace_panic_trigger)
3223
3224 #elif defined(__i386)
3225
3226 ENTRY_NP(panic_trigger)
3227 movl 4(%esp), %edx / %edx = address of trigger
3228 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3229 lock / assert lock
3230 xchgl %eax, (%edx) / exchange %eax and the trigger
3231 cmpl $0, %eax / if (%eax == 0x0)
3232 je 0f / return (1);
3233 movl $0, %eax / else
3234 ret / return (0);
3235 0: movl $1, %eax
3236 ret
3237 SET_SIZE(panic_trigger)
3238
3239 ENTRY_NP(dtrace_panic_trigger)
3240 movl 4(%esp), %edx / %edx = address of trigger
3241 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3242 lock / assert lock
3243 xchgl %eax, (%edx) / exchange %eax and the trigger
3244 cmpl $0, %eax / if (%eax == 0x0)
3245 je 0f / return (1);
3246 movl $0, %eax / else
3247 ret / return (0);
3248 0: movl $1, %eax
3249 ret
3250 SET_SIZE(dtrace_panic_trigger)
3251
3252 #endif /* __i386 */
3253 #endif /* __lint */
3254
3255 /*
3256 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3257 * into the panic code implemented in panicsys(). vpanic() is responsible
3258 * for passing through the format string and arguments, and constructing a
3259 * regs structure on the stack into which it saves the current register
3260 * values. If we are not dying due to a fatal trap, these registers will
3261 * then be preserved in panicbuf as the current processor state. Before
3262 * invoking panicsys(), vpanic() activates the first panic trigger (see
3263 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3264 * DTrace takes a slightly different panic path if it must panic from probe
3265 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3266 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3267 * branches back into vpanic().
3268 */
3269 #if defined(__lint)
3270
3271 /*ARGSUSED*/
3272 void
3273 vpanic(const char *format, va_list alist)
3274 {}
3275
3276 /*ARGSUSED*/
3277 void
3278 dtrace_vpanic(const char *format, va_list alist)
3279 {}
3280
3281 #else /* __lint */
3282
3283 #if defined(__amd64)
3284
3285 ENTRY_NP(vpanic) /* Initial stack layout: */
3286
3287 pushq %rbp /* | %rip | 0x60 */
3288 movq %rsp, %rbp /* | %rbp | 0x58 */
3289 pushfq /* | rfl | 0x50 */
3290 pushq %r11 /* | %r11 | 0x48 */
3291 pushq %r10 /* | %r10 | 0x40 */
3292 pushq %rbx /* | %rbx | 0x38 */
3293 pushq %rax /* | %rax | 0x30 */
3294 pushq %r9 /* | %r9 | 0x28 */
3295 pushq %r8 /* | %r8 | 0x20 */
3296 pushq %rcx /* | %rcx | 0x18 */
3297 pushq %rdx /* | %rdx | 0x10 */
3298 pushq %rsi /* | %rsi | 0x8 alist */
3299 pushq %rdi /* | %rdi | 0x0 format */
3300
3301 movq %rsp, %rbx /* %rbx = current %rsp */
3302
3303 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3304 call panic_trigger /* %eax = panic_trigger() */
3305
3306 vpanic_common:
3307 /*
3308 * The panic_trigger result is in %eax from the call above, and
3309 * dtrace_panic places it in %eax before branching here.
3310 * The rdmsr instructions that follow below will clobber %eax so
3311 * we stash the panic_trigger result in %r11d.
3312 */
3313 movl %eax, %r11d
3314 cmpl $0, %r11d
3315 je 0f
3316
3317 /*
3318 * If panic_trigger() was successful, we are the first to initiate a
3319 * panic: we now switch to the reserved panic_stack before continuing.
3320 */
3321 leaq panic_stack(%rip), %rsp
3322 addq $PANICSTKSIZE, %rsp
3323 0: subq $REGSIZE, %rsp
3324 /*
3325 * Now that we've got everything set up, store the register values as
3326 * they were when we entered vpanic() to the designated location in
3327 * the regs structure we allocated on the stack.
3328 */
3329 movq 0x0(%rbx), %rcx
3330 movq %rcx, REGOFF_RDI(%rsp)
3331 movq 0x8(%rbx), %rcx
3332 movq %rcx, REGOFF_RSI(%rsp)
3333 movq 0x10(%rbx), %rcx
3334 movq %rcx, REGOFF_RDX(%rsp)
3335 movq 0x18(%rbx), %rcx
3336 movq %rcx, REGOFF_RCX(%rsp)
3337 movq 0x20(%rbx), %rcx
3338
3339 movq %rcx, REGOFF_R8(%rsp)
3340 movq 0x28(%rbx), %rcx
3341 movq %rcx, REGOFF_R9(%rsp)
3342 movq 0x30(%rbx), %rcx
3343 movq %rcx, REGOFF_RAX(%rsp)
3344 movq 0x38(%rbx), %rcx
3345 movq %rcx, REGOFF_RBX(%rsp)
3346 movq 0x58(%rbx), %rcx
3347
3348 movq %rcx, REGOFF_RBP(%rsp)
3349 movq 0x40(%rbx), %rcx
3350 movq %rcx, REGOFF_R10(%rsp)
3351 movq 0x48(%rbx), %rcx
3352 movq %rcx, REGOFF_R11(%rsp)
3353 movq %r12, REGOFF_R12(%rsp)
3354
3355 movq %r13, REGOFF_R13(%rsp)
3356 movq %r14, REGOFF_R14(%rsp)
3357 movq %r15, REGOFF_R15(%rsp)
3358
3359 xorl %ecx, %ecx
3360 movw %ds, %cx
3361 movq %rcx, REGOFF_DS(%rsp)
3362 movw %es, %cx
3363 movq %rcx, REGOFF_ES(%rsp)
3364 movw %fs, %cx
3365 movq %rcx, REGOFF_FS(%rsp)
3366 movw %gs, %cx
3367 movq %rcx, REGOFF_GS(%rsp)
3368
3369 movq $0, REGOFF_TRAPNO(%rsp)
3370
3371 movq $0, REGOFF_ERR(%rsp)
3372 leaq vpanic(%rip), %rcx
3373 movq %rcx, REGOFF_RIP(%rsp)
3374 movw %cs, %cx
3375 movzwq %cx, %rcx
3376 movq %rcx, REGOFF_CS(%rsp)
3377 movq 0x50(%rbx), %rcx
3378 movq %rcx, REGOFF_RFL(%rsp)
3379 movq %rbx, %rcx
3380 addq $0x60, %rcx
3381 movq %rcx, REGOFF_RSP(%rsp)
3382 movw %ss, %cx
3383 movzwq %cx, %rcx
3384 movq %rcx, REGOFF_SS(%rsp)
3385
3386 /*
3387 * panicsys(format, alist, rp, on_panic_stack)
3388 */
3389 movq REGOFF_RDI(%rsp), %rdi /* format */
3390 movq REGOFF_RSI(%rsp), %rsi /* alist */
3391 movq %rsp, %rdx /* struct regs */
3392 movl %r11d, %ecx /* on_panic_stack */
3393 call panicsys
3394 addq $REGSIZE, %rsp
3395 popq %rdi
3396 popq %rsi
3397 popq %rdx
3398 popq %rcx
3399 popq %r8
3400 popq %r9
3401 popq %rax
3402 popq %rbx
3403 popq %r10
3404 popq %r11
3405 popfq
3406 leave
3407 ret
3408 SET_SIZE(vpanic)
3409
3410 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3411
3412 pushq %rbp /* | %rip | 0x60 */
3413 movq %rsp, %rbp /* | %rbp | 0x58 */
3414 pushfq /* | rfl | 0x50 */
3415 pushq %r11 /* | %r11 | 0x48 */
3416 pushq %r10 /* | %r10 | 0x40 */
3417 pushq %rbx /* | %rbx | 0x38 */
3418 pushq %rax /* | %rax | 0x30 */
3419 pushq %r9 /* | %r9 | 0x28 */
3420 pushq %r8 /* | %r8 | 0x20 */
3421 pushq %rcx /* | %rcx | 0x18 */
3422 pushq %rdx /* | %rdx | 0x10 */
3423 pushq %rsi /* | %rsi | 0x8 alist */
3424 pushq %rdi /* | %rdi | 0x0 format */
3425
3426 movq %rsp, %rbx /* %rbx = current %rsp */
3427
3428 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3429 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3430 jmp vpanic_common
3431
3432 SET_SIZE(dtrace_vpanic)
3433
3434 #elif defined(__i386)
3435
3436 ENTRY_NP(vpanic) / Initial stack layout:
3437
3438 pushl %ebp / | %eip | 20
3439 movl %esp, %ebp / | %ebp | 16
3440 pushl %eax / | %eax | 12
3441 pushl %ebx / | %ebx | 8
3442 pushl %ecx / | %ecx | 4
3443 pushl %edx / | %edx | 0
3444
3445 movl %esp, %ebx / %ebx = current stack pointer
3446
3447 lea panic_quiesce, %eax / %eax = &panic_quiesce
3448 pushl %eax / push &panic_quiesce
3449 call panic_trigger / %eax = panic_trigger()
3450 addl $4, %esp / reset stack pointer
3451
3452 vpanic_common:
3453 cmpl $0, %eax / if (%eax == 0)
3454 je 0f / goto 0f;
3455
3456 /*
3457 * If panic_trigger() was successful, we are the first to initiate a
3458 * panic: we now switch to the reserved panic_stack before continuing.
3459 */
3460 lea panic_stack, %esp / %esp = panic_stack
3461 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3462
3463 0: subl $REGSIZE, %esp / allocate struct regs
3464
3465 /*
3466 * Now that we've got everything set up, store the register values as
3467 * they were when we entered vpanic() to the designated location in
3468 * the regs structure we allocated on the stack.
3469 */
3470 #if !defined(__GNUC_AS__)
3471 movw %gs, %edx
3472 movl %edx, REGOFF_GS(%esp)
3473 movw %fs, %edx
3474 movl %edx, REGOFF_FS(%esp)
3475 movw %es, %edx
3476 movl %edx, REGOFF_ES(%esp)
3477 movw %ds, %edx
3478 movl %edx, REGOFF_DS(%esp)
3479 #else /* __GNUC_AS__ */
3480 mov %gs, %edx
3481 mov %edx, REGOFF_GS(%esp)
3482 mov %fs, %edx
3483 mov %edx, REGOFF_FS(%esp)
3484 mov %es, %edx
3485 mov %edx, REGOFF_ES(%esp)
3486 mov %ds, %edx
3487 mov %edx, REGOFF_DS(%esp)
3488 #endif /* __GNUC_AS__ */
3489 movl %edi, REGOFF_EDI(%esp)
3490 movl %esi, REGOFF_ESI(%esp)
3491 movl 16(%ebx), %ecx
3492 movl %ecx, REGOFF_EBP(%esp)
3493 movl %ebx, %ecx
3494 addl $20, %ecx
3495 movl %ecx, REGOFF_ESP(%esp)
3496 movl 8(%ebx), %ecx
3497 movl %ecx, REGOFF_EBX(%esp)
3498 movl 0(%ebx), %ecx
3499 movl %ecx, REGOFF_EDX(%esp)
3500 movl 4(%ebx), %ecx
3501 movl %ecx, REGOFF_ECX(%esp)
3502 movl 12(%ebx), %ecx
3503 movl %ecx, REGOFF_EAX(%esp)
3504 movl $0, REGOFF_TRAPNO(%esp)
3505 movl $0, REGOFF_ERR(%esp)
3506 lea vpanic, %ecx
3507 movl %ecx, REGOFF_EIP(%esp)
3508 #if !defined(__GNUC_AS__)
3509 movw %cs, %edx
3510 #else /* __GNUC_AS__ */
3511 mov %cs, %edx
3512 #endif /* __GNUC_AS__ */
3513 movl %edx, REGOFF_CS(%esp)
3514 pushfl
3515 popl %ecx
3516 #if defined(__xpv)
3517 /*
3518 * Synthesize the PS_IE bit from the event mask bit
3519 */
3520 CURTHREAD(%edx)
3521 KPREEMPT_DISABLE(%edx)
3522 EVENT_MASK_TO_IE(%edx, %ecx)
3523 CURTHREAD(%edx)
3524 KPREEMPT_ENABLE_NOKP(%edx)
3525 #endif
3526 movl %ecx, REGOFF_EFL(%esp)
3527 movl $0, REGOFF_UESP(%esp)
3528 #if !defined(__GNUC_AS__)
3529 movw %ss, %edx
3530 #else /* __GNUC_AS__ */
3531 mov %ss, %edx
3532 #endif /* __GNUC_AS__ */
3533 movl %edx, REGOFF_SS(%esp)
3534
3535 movl %esp, %ecx / %ecx = ®s
3536 pushl %eax / push on_panic_stack
3537 pushl %ecx / push ®s
3538 movl 12(%ebp), %ecx / %ecx = alist
3539 pushl %ecx / push alist
3540 movl 8(%ebp), %ecx / %ecx = format
3541 pushl %ecx / push format
3542 call panicsys / panicsys();
3543 addl $16, %esp / pop arguments
3544
3545 addl $REGSIZE, %esp
3546 popl %edx
3547 popl %ecx
3548 popl %ebx
3549 popl %eax
3550 leave
3551 ret
3552 SET_SIZE(vpanic)
3553
3554 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3555
3556 pushl %ebp / | %eip | 20
3557 movl %esp, %ebp / | %ebp | 16
3558 pushl %eax / | %eax | 12
3559 pushl %ebx / | %ebx | 8
3560 pushl %ecx / | %ecx | 4
3561 pushl %edx / | %edx | 0
3562
3563 movl %esp, %ebx / %ebx = current stack pointer
3564
3565 lea panic_quiesce, %eax / %eax = &panic_quiesce
3566 pushl %eax / push &panic_quiesce
3567 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3568 addl $4, %esp / reset stack pointer
3569 jmp vpanic_common / jump back to common code
3570
3571 SET_SIZE(dtrace_vpanic)
3572
3573 #endif /* __i386 */
3574 #endif /* __lint */
3575
3576 #if defined(__lint)
3577
3578 void
3579 hres_tick(void)
3580 {}
3581
3582 int64_t timedelta;
3583 hrtime_t hrtime_base;
3584
3585 #else /* __lint */
3586
3587 DGDEF3(timedelta, 8, 8)
3588 .long 0, 0
3589
3590 /*
3591 * initialized to a non zero value to make pc_gethrtime()
3592 * work correctly even before clock is initialized
3593 */
3594 DGDEF3(hrtime_base, 8, 8)
3595 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3596
3597 DGDEF3(adj_shift, 4, 4)
3598 .long ADJ_SHIFT
3599
3600 #if defined(__amd64)
3601
3602 ENTRY_NP(hres_tick)
3603 pushq %rbp
3604 movq %rsp, %rbp
3605
3606 /*
3607 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3608 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3609 * At worst, performing this now instead of under CLOCK_LOCK may
3610 * introduce some jitter in pc_gethrestime().
3611 */
3612 call *gethrtimef(%rip)
3613 movq %rax, %r8
3614
3615 leaq hres_lock(%rip), %rax
3616 movb $-1, %dl
3617 .CL1:
3618 xchgb %dl, (%rax)
3619 testb %dl, %dl
3620 jz .CL3 /* got it */
3621 .CL2:
3622 cmpb $0, (%rax) /* possible to get lock? */
3623 pause
3624 jne .CL2
3625 jmp .CL1 /* yes, try again */
3626 .CL3:
3627 /*
3628 * compute the interval since last time hres_tick was called
3629 * and adjust hrtime_base and hrestime accordingly
3630 * hrtime_base is an 8 byte value (in nsec), hrestime is
3631 * a timestruc_t (sec, nsec)
3632 */
3633 leaq hres_last_tick(%rip), %rax
3634 movq %r8, %r11
3635 subq (%rax), %r8
3636 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3637 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3638 /*
3639 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3640 */
3641 movq %r11, (%rax)
3642
3643 call __adj_hrestime
3644
3645 /*
3646 * release the hres_lock
3647 */
3648 incl hres_lock(%rip)
3649 leave
3650 ret
3651 SET_SIZE(hres_tick)
3652
3653 #elif defined(__i386)
3654
3655 ENTRY_NP(hres_tick)
3656 pushl %ebp
3657 movl %esp, %ebp
3658 pushl %esi
3659 pushl %ebx
3660
3661 /*
3662 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3663 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3664 * At worst, performing this now instead of under CLOCK_LOCK may
3665 * introduce some jitter in pc_gethrestime().
3666 */
3667 call *gethrtimef
3668 movl %eax, %ebx
3669 movl %edx, %esi
3670
3671 movl $hres_lock, %eax
3672 movl $-1, %edx
3673 .CL1:
3674 xchgb %dl, (%eax)
3675 testb %dl, %dl
3676 jz .CL3 / got it
3677 .CL2:
3678 cmpb $0, (%eax) / possible to get lock?
3679 pause
3680 jne .CL2
3681 jmp .CL1 / yes, try again
3682 .CL3:
3683 /*
3684 * compute the interval since last time hres_tick was called
3685 * and adjust hrtime_base and hrestime accordingly
3686 * hrtime_base is an 8 byte value (in nsec), hrestime is
3687 * timestruc_t (sec, nsec)
3688 */
3689
3690 lea hres_last_tick, %eax
3691
3692 movl %ebx, %edx
3693 movl %esi, %ecx
3694
3695 subl (%eax), %edx
3696 sbbl 4(%eax), %ecx
3697
3698 addl %edx, hrtime_base / add interval to hrtime_base
3699 adcl %ecx, hrtime_base+4
3700
3701 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3702
3703 /
3704 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3705 /
3706 movl %ebx, (%eax)
3707 movl %esi, 4(%eax)
3708
3709 / get hrestime at this moment. used as base for pc_gethrestime
3710 /
3711 / Apply adjustment, if any
3712 /
3713 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3714 / (max_hres_adj)
3715 /
3716 / void
3717 / adj_hrestime()
3718 / {
3719 / long long adj;
3720 /
3721 / if (hrestime_adj == 0)
3722 / adj = 0;
3723 / else if (hrestime_adj > 0) {
3724 / if (hrestime_adj < HRES_ADJ)
3725 / adj = hrestime_adj;
3726 / else
3727 / adj = HRES_ADJ;
3728 / }
3729 / else {
3730 / if (hrestime_adj < -(HRES_ADJ))
3731 / adj = -(HRES_ADJ);
3732 / else
3733 / adj = hrestime_adj;
3734 / }
3735 /
3736 / timedelta -= adj;
3737 / hrestime_adj = timedelta;
3738 / hrestime.tv_nsec += adj;
3739 /
3740 / while (hrestime.tv_nsec >= NANOSEC) {
3741 / one_sec++;
3742 / hrestime.tv_sec++;
3743 / hrestime.tv_nsec -= NANOSEC;
3744 / }
3745 / }
3746 __adj_hrestime:
3747 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3748 movl hrestime_adj+4, %edx
3749 andl %esi, %esi
3750 jne .CL4 / no
3751 andl %edx, %edx
3752 jne .CL4 / no
3753 subl %ecx, %ecx / yes, adj = 0;
3754 subl %edx, %edx
3755 jmp .CL5
3756 .CL4:
3757 subl %ecx, %ecx
3758 subl %eax, %eax
3759 subl %esi, %ecx
3760 sbbl %edx, %eax
3761 andl %eax, %eax / if (hrestime_adj > 0)
3762 jge .CL6
3763
3764 / In the following comments, HRES_ADJ is used, while in the code
3765 / max_hres_adj is used.
3766 /
3767 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3768 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3769 / on the logical equivalence of:
3770 /
3771 / !(hrestime_adj < HRES_ADJ)
3772 /
3773 / and the two step sequence:
3774 /
3775 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3776 /
3777 / which computes whether or not the least significant 32-bits
3778 / of hrestime_adj is greater than HRES_ADJ, followed by:
3779 /
3780 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3781 /
3782 / which generates a carry whenever step 1 is true or the most
3783 / significant long of the longlong hrestime_adj is non-zero.
3784
3785 movl max_hres_adj, %ecx / hrestime_adj is positive
3786 subl %esi, %ecx
3787 movl %edx, %eax
3788 adcl $-1, %eax
3789 jnc .CL7
3790 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3791 subl %edx, %edx
3792 jmp .CL5
3793
3794 / The following computation is similar to the one above.
3795 /
3796 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3797 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3798 / on the logical equivalence of:
3799 /
3800 / (hrestime_adj > -HRES_ADJ)
3801 /
3802 / and the two step sequence:
3803 /
3804 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3805 /
3806 / which means the least significant 32-bits of hrestime_adj is
3807 / greater than -HRES_ADJ, followed by:
3808 /
3809 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3810 /
3811 / which generates a carry only when step 1 is true and the most
3812 / significant long of the longlong hrestime_adj is -1.
3813
3814 .CL6: / hrestime_adj is negative
3815 movl %esi, %ecx
3816 addl max_hres_adj, %ecx
3817 movl %edx, %eax
3818 adcl $0, %eax
3819 jc .CL7
3820 xor %ecx, %ecx
3821 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3822 movl $-1, %edx
3823 jmp .CL5
3824 .CL7:
3825 movl %esi, %ecx / adj = hrestime_adj;
3826 .CL5:
3827 movl timedelta, %esi
3828 subl %ecx, %esi
3829 movl timedelta+4, %eax
3830 sbbl %edx, %eax
3831 movl %esi, timedelta
3832 movl %eax, timedelta+4 / timedelta -= adj;
3833 movl %esi, hrestime_adj
3834 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3835 addl hrestime+4, %ecx
3836
3837 movl %ecx, %eax / eax = tv_nsec
3838 1:
3839 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3840 jb .CL8 / no
3841 incl one_sec / yes, one_sec++;
3842 incl hrestime / hrestime.tv_sec++;
3843 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3844 jmp 1b / check for more seconds
3845
3846 .CL8:
3847 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3848 incl hres_lock / release the hres_lock
3849
3850 popl %ebx
3851 popl %esi
3852 leave
3853 ret
3854 SET_SIZE(hres_tick)
3855
3856 #endif /* __i386 */
3857 #endif /* __lint */
3858
3859 /*
3860 * void prefetch_smap_w(void *)
3861 *
3862 * Prefetch ahead within a linear list of smap structures.
3863 * Not implemented for ia32. Stub for compatibility.
3864 */
3865
3866 #if defined(__lint)
3867
3868 /*ARGSUSED*/
3869 void prefetch_smap_w(void *smp)
3870 {}
3871
3872 #else /* __lint */
3873
3874 ENTRY(prefetch_smap_w)
3875 rep; ret /* use 2 byte return instruction when branch target */
3876 /* AMD Software Optimization Guide - Section 6.2 */
3877 SET_SIZE(prefetch_smap_w)
3878
3879 #endif /* __lint */
3880
3881 /*
3882 * prefetch_page_r(page_t *)
3883 * issue prefetch instructions for a page_t
3884 */
3885 #if defined(__lint)
3886
3887 /*ARGSUSED*/
3888 void
3889 prefetch_page_r(void *pp)
3890 {}
3891
3892 #else /* __lint */
3893
3894 ENTRY(prefetch_page_r)
3895 rep; ret /* use 2 byte return instruction when branch target */
3896 /* AMD Software Optimization Guide - Section 6.2 */
3897 SET_SIZE(prefetch_page_r)
3898
3899 #endif /* __lint */
3900
3901 #if defined(__lint)
3902
3903 /*ARGSUSED*/
3904 int
3905 bcmp(const void *s1, const void *s2, size_t count)
3906 { return (0); }
3907
3908 #else /* __lint */
3909
3910 #if defined(__amd64)
3911
3912 ENTRY(bcmp)
3913 pushq %rbp
3914 movq %rsp, %rbp
3915 #ifdef DEBUG
3916 testq %rdx,%rdx
3917 je 1f
3918 movq postbootkernelbase(%rip), %r11
3919 cmpq %r11, %rdi
3920 jb 0f
3921 cmpq %r11, %rsi
3922 jnb 1f
3923 0: leaq .bcmp_panic_msg(%rip), %rdi
3924 xorl %eax, %eax
3925 call panic
3926 1:
3927 #endif /* DEBUG */
3928 call memcmp
3929 testl %eax, %eax
3930 setne %dl
3931 leave
3932 movzbl %dl, %eax
3933 ret
3934 SET_SIZE(bcmp)
3935
3936 #elif defined(__i386)
3937
3938 #define ARG_S1 8
3939 #define ARG_S2 12
3940 #define ARG_LENGTH 16
3941
3942 ENTRY(bcmp)
3943 pushl %ebp
3944 movl %esp, %ebp / create new stack frame
3945 #ifdef DEBUG
3946 cmpl $0, ARG_LENGTH(%ebp)
3947 je 1f
3948 movl postbootkernelbase, %eax
3949 cmpl %eax, ARG_S1(%ebp)
3950 jb 0f
3951 cmpl %eax, ARG_S2(%ebp)
3952 jnb 1f
3953 0: pushl $.bcmp_panic_msg
3954 call panic
3955 1:
3956 #endif /* DEBUG */
3957
3958 pushl %edi / save register variable
3959 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3960 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3961 cmpl %eax, %ecx / if the same string
3962 je .equal / goto .equal
3963 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3964 cmpl $4, %edi / if %edi < 4
3965 jb .byte_check / goto .byte_check
3966 .align 4
3967 .word_loop:
3968 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3969 leal -4(%edi), %edi / %edi -= 4
3970 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3971 jne .word_not_equal / if not equal, goto .word_not_equal
3972 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3973 leal 4(%eax), %eax / %eax += 4 (next word)
3974 cmpl $4, %edi / if %edi >= 4
3975 jae .word_loop / goto .word_loop
3976 .byte_check:
3977 cmpl $0, %edi / if %edi == 0
3978 je .equal / goto .equal
3979 jmp .byte_loop / goto .byte_loop (checks in bytes)
3980 .word_not_equal:
3981 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3982 .align 4
3983 .byte_loop:
3984 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3985 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3986 jne .not_equal / if not equal, goto .not_equal
3987 incl %ecx / %ecx++ (next byte)
3988 incl %eax / %eax++ (next byte)
3989 decl %edi / %edi--
3990 jnz .byte_loop / if not zero, goto .byte_loop
3991 .equal:
3992 xorl %eax, %eax / %eax = 0
3993 popl %edi / restore register variable
3994 leave / restore old stack frame
3995 ret / return (NULL)
3996 .align 4
3997 .not_equal:
3998 movl $1, %eax / return 1
3999 popl %edi / restore register variable
4000 leave / restore old stack frame
4001 ret / return (NULL)
4002 SET_SIZE(bcmp)
4003
4004 #endif /* __i386 */
4005
4006 #ifdef DEBUG
4007 .text
4008 .bcmp_panic_msg:
4009 .string "bcmp: arguments below kernelbase"
4010 #endif /* DEBUG */
4011
4012 #endif /* __lint */
4013
4014 #if defined(__lint)
4015
4016 uint_t
4017 bsrw_insn(uint16_t mask)
4018 {
4019 uint_t index = sizeof (mask) * NBBY - 1;
4020
4021 while ((mask & (1 << index)) == 0)
4022 index--;
4023 return (index);
4024 }
4025
4026 #else /* __lint */
4027
4028 #if defined(__amd64)
4029
4030 ENTRY_NP(bsrw_insn)
4031 xorl %eax, %eax
4032 bsrw %di, %ax
4033 ret
4034 SET_SIZE(bsrw_insn)
4035
4036 #elif defined(__i386)
4037
4038 ENTRY_NP(bsrw_insn)
4039 movw 4(%esp), %cx
4040 xorl %eax, %eax
4041 bsrw %cx, %ax
4042 ret
4043 SET_SIZE(bsrw_insn)
4044
4045 #endif /* __i386 */
4046 #endif /* __lint */
4047
4048 #if defined(__lint)
4049
4050 uint_t
4051 atomic_btr32(uint32_t *pending, uint_t pil)
4052 {
4053 return (*pending &= ~(1 << pil));
4054 }
4055
4056 #else /* __lint */
4057
4058 #if defined(__i386)
4059
4060 ENTRY_NP(atomic_btr32)
4061 movl 4(%esp), %ecx
4062 movl 8(%esp), %edx
4063 xorl %eax, %eax
4064 lock
4065 btrl %edx, (%ecx)
4066 setc %al
4067 ret
4068 SET_SIZE(atomic_btr32)
4069
4070 #endif /* __i386 */
4071 #endif /* __lint */
4072
4073 #if defined(__lint)
4074
4075 /*ARGSUSED*/
4076 void
4077 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4078 uint_t arg2)
4079 {}
4080
4081 #else /* __lint */
4082
4083 #if defined(__amd64)
4084
4085 ENTRY_NP(switch_sp_and_call)
4086 pushq %rbp
4087 movq %rsp, %rbp /* set up stack frame */
4088 movq %rdi, %rsp /* switch stack pointer */
4089 movq %rdx, %rdi /* pass func arg 1 */
4090 movq %rsi, %r11 /* save function to call */
4091 movq %rcx, %rsi /* pass func arg 2 */
4092 call *%r11 /* call function */
4093 leave /* restore stack */
4094 ret
4095 SET_SIZE(switch_sp_and_call)
4096
4097 #elif defined(__i386)
4098
4099 ENTRY_NP(switch_sp_and_call)
4100 pushl %ebp
4101 mov %esp, %ebp /* set up stack frame */
4102 movl 8(%ebp), %esp /* switch stack pointer */
4103 pushl 20(%ebp) /* push func arg 2 */
4104 pushl 16(%ebp) /* push func arg 1 */
4105 call *12(%ebp) /* call function */
4106 addl $8, %esp /* pop arguments */
4107 leave /* restore stack */
4108 ret
4109 SET_SIZE(switch_sp_and_call)
4110
4111 #endif /* __i386 */
4112 #endif /* __lint */
4113
4114 #if defined(__lint)
4115
4116 void
4117 kmdb_enter(void)
4118 {}
4119
4120 #else /* __lint */
4121
4122 #if defined(__amd64)
4123
4124 ENTRY_NP(kmdb_enter)
4125 pushq %rbp
4126 movq %rsp, %rbp
4127
4128 /*
4129 * Save flags, do a 'cli' then return the saved flags
4130 */
4131 call intr_clear
4132
4133 int $T_DBGENTR
4134
4135 /*
4136 * Restore the saved flags
4137 */
4138 movq %rax, %rdi
4139 call intr_restore
4140
4141 leave
4142 ret
4143 SET_SIZE(kmdb_enter)
4144
4145 #elif defined(__i386)
4146
4147 ENTRY_NP(kmdb_enter)
4148 pushl %ebp
4149 movl %esp, %ebp
4150
4151 /*
4152 * Save flags, do a 'cli' then return the saved flags
4153 */
4154 call intr_clear
4155
4156 int $T_DBGENTR
4157
4158 /*
4159 * Restore the saved flags
4160 */
4161 pushl %eax
4162 call intr_restore
4163 addl $4, %esp
4164
4165 leave
4166 ret
4167 SET_SIZE(kmdb_enter)
4168
4169 #endif /* __i386 */
4170 #endif /* __lint */
4171
4172 #if defined(__lint)
4173
4174 void
4175 return_instr(void)
4176 {}
4177
4178 #else /* __lint */
4179
4180 ENTRY_NP(return_instr)
4181 rep; ret /* use 2 byte instruction when branch target */
4182 /* AMD Software Optimization Guide - Section 6.2 */
4183 SET_SIZE(return_instr)
4184
4185 #endif /* __lint */
4186
4187 #if defined(__lint)
4188
4189 ulong_t
4190 getflags(void)
4191 {
4192 return (0);
4193 }
4194
4195 #else /* __lint */
4196
4197 #if defined(__amd64)
4198
4199 ENTRY(getflags)
4200 pushfq
4201 popq %rax
4202 #if defined(__xpv)
4203 CURTHREAD(%rdi)
4204 KPREEMPT_DISABLE(%rdi)
4205 /*
4206 * Synthesize the PS_IE bit from the event mask bit
4207 */
4208 CURVCPU(%r11)
4209 andq $_BITNOT(PS_IE), %rax
4210 XEN_TEST_UPCALL_MASK(%r11)
4211 jnz 1f
4212 orq $PS_IE, %rax
4213 1:
4214 KPREEMPT_ENABLE_NOKP(%rdi)
4215 #endif
4216 ret
4217 SET_SIZE(getflags)
4218
4219 #elif defined(__i386)
4220
4221 ENTRY(getflags)
4222 pushfl
4223 popl %eax
4224 #if defined(__xpv)
4225 CURTHREAD(%ecx)
4226 KPREEMPT_DISABLE(%ecx)
4227 /*
4228 * Synthesize the PS_IE bit from the event mask bit
4229 */
4230 CURVCPU(%edx)
4231 andl $_BITNOT(PS_IE), %eax
4232 XEN_TEST_UPCALL_MASK(%edx)
4233 jnz 1f
4234 orl $PS_IE, %eax
4235 1:
4236 KPREEMPT_ENABLE_NOKP(%ecx)
4237 #endif
4238 ret
4239 SET_SIZE(getflags)
4240
4241 #endif /* __i386 */
4242
4243 #endif /* __lint */
4244
4245 #if defined(__lint)
4246
4247 ftrace_icookie_t
4248 ftrace_interrupt_disable(void)
4249 { return (0); }
4250
4251 #else /* __lint */
4252
4253 #if defined(__amd64)
4254
4255 ENTRY(ftrace_interrupt_disable)
4256 pushfq
4257 popq %rax
4258 CLI(%rdx)
4259 ret
4260 SET_SIZE(ftrace_interrupt_disable)
4261
4262 #elif defined(__i386)
4263
4264 ENTRY(ftrace_interrupt_disable)
4265 pushfl
4266 popl %eax
4267 CLI(%edx)
4268 ret
4269 SET_SIZE(ftrace_interrupt_disable)
4270
4271 #endif /* __i386 */
4272 #endif /* __lint */
4273
4274 #if defined(__lint)
4275
4276 /*ARGSUSED*/
4277 void
4278 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4279 {}
4280
4281 #else /* __lint */
4282
4283 #if defined(__amd64)
4284
4285 ENTRY(ftrace_interrupt_enable)
4286 pushq %rdi
4287 popfq
4288 ret
4289 SET_SIZE(ftrace_interrupt_enable)
4290
4291 #elif defined(__i386)
4292
4293 ENTRY(ftrace_interrupt_enable)
4294 movl 4(%esp), %eax
4295 pushl %eax
4296 popfl
4297 ret
4298 SET_SIZE(ftrace_interrupt_enable)
4299
4300 #endif /* __i386 */
4301 #endif /* __lint */
4302
4303 #if defined (__lint)
4304
4305 /*ARGSUSED*/
4306 void
4307 clflush_insn(caddr_t addr)
4308 {}
4309
4310 #else /* __lint */
4311
4312 #if defined (__amd64)
4313 ENTRY(clflush_insn)
4314 clflush (%rdi)
4315 ret
4316 SET_SIZE(clflush_insn)
4317 #elif defined (__i386)
4318 ENTRY(clflush_insn)
4319 movl 4(%esp), %eax
4320 clflush (%eax)
4321 ret
4322 SET_SIZE(clflush_insn)
4323
4324 #endif /* __i386 */
4325 #endif /* __lint */
4326
4327 #if defined (__lint)
4328 /*ARGSUSED*/
4329 void
4330 mfence_insn(void)
4331 {}
4332
4333 #else /* __lint */
4334
4335 #if defined (__amd64)
4336 ENTRY(mfence_insn)
4337 mfence
4338 ret
4339 SET_SIZE(mfence_insn)
4340 #elif defined (__i386)
4341 ENTRY(mfence_insn)
4342 mfence
4343 ret
4344 SET_SIZE(mfence_insn)
4345
4346 #endif /* __i386 */
4347 #endif /* __lint */
4348
4349 /*
4350 * VMware implements an I/O port that programs can query to detect if software
4351 * is running in a VMware hypervisor. This hypervisor port behaves differently
4352 * depending on magic values in certain registers and modifies some registers
4353 * as a side effect.
4354 *
4355 * References: http://kb.vmware.com/kb/1009458
4356 */
4357
4358 #if defined(__lint)
4359
4360 /* ARGSUSED */
4361 void
4362 vmware_port(int cmd, uint32_t *regs) { return; }
4363
4364 #else
4365
4366 #if defined(__amd64)
4367
4368 ENTRY(vmware_port)
4369 pushq %rbx
4370 movl $VMWARE_HVMAGIC, %eax
4371 movl $0xffffffff, %ebx
4372 movl %edi, %ecx
4373 movl $VMWARE_HVPORT, %edx
4374 inl (%dx)
4375 movl %eax, (%rsi)
4376 movl %ebx, 4(%rsi)
4377 movl %ecx, 8(%rsi)
4378 movl %edx, 12(%rsi)
4379 popq %rbx
4380 ret
4381 SET_SIZE(vmware_port)
4382
4383 #elif defined(__i386)
4384
4385 ENTRY(vmware_port)
4386 pushl %ebx
4387 pushl %esi
4388 movl $VMWARE_HVMAGIC, %eax
4389 movl $0xffffffff, %ebx
4390 movl 12(%esp), %ecx
4391 movl $VMWARE_HVPORT, %edx
4392 inl (%dx)
4393 movl 16(%esp), %esi
4394 movl %eax, (%esi)
4395 movl %ebx, 4(%esi)
4396 movl %ecx, 8(%esi)
4397 movl %edx, 12(%esi)
4398 popl %esi
4399 popl %ebx
4400 ret
4401 SET_SIZE(vmware_port)
4402
4403 #endif /* __i386 */
4404 #endif /* __lint */