Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/i86_subr.s
+++ new/usr/src/uts/intel/ia32/ml/i86_subr.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 - * Copyright 2016 Joyent, Inc.
26 + * Copyright 2018 Joyent, Inc.
27 27 */
28 28
29 29 /*
30 30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 32 * All Rights Reserved
33 33 */
34 34
35 35 /*
36 36 * Copyright (c) 2009, Intel Corporation.
37 37 * All rights reserved.
38 38 */
39 39
40 40 /*
41 41 * General assembly language routines.
42 42 * It is the intent of this file to contain routines that are
43 43 * independent of the specific kernel architecture, and those that are
44 44 * common across kernel architectures.
45 45 * As architectures diverge, and implementations of specific
46 46 * architecture-dependent routines change, the routines should be moved
47 47 * from this file into the respective ../`arch -k`/subr.s file.
48 48 */
49 49
50 50 #include <sys/asm_linkage.h>
51 51 #include <sys/asm_misc.h>
52 52 #include <sys/panic.h>
53 53 #include <sys/ontrap.h>
54 54 #include <sys/regset.h>
55 55 #include <sys/privregs.h>
56 56 #include <sys/reboot.h>
57 57 #include <sys/psw.h>
58 58 #include <sys/x86_archext.h>
59 59
60 60 #if defined(__lint)
61 61 #include <sys/types.h>
62 62 #include <sys/systm.h>
63 63 #include <sys/thread.h>
64 64 #include <sys/archsystm.h>
65 65 #include <sys/byteorder.h>
66 66 #include <sys/dtrace.h>
67 67 #include <sys/ftrace.h>
68 68 #else /* __lint */
69 69 #include "assym.h"
70 70 #endif /* __lint */
71 71 #include <sys/dditypes.h>
72 72
73 73 /*
74 74 * on_fault()
75 75 *
76 76 * Catch lofault faults. Like setjmp except it returns one
77 77 * if code following causes uncorrectable fault. Turned off
78 78 * by calling no_fault(). Note that while under on_fault(),
79 79 * SMAP is disabled. For more information see
80 80 * uts/intel/ia32/ml/copy.s.
81 81 */
82 82
83 83 #if defined(__lint)
84 84
85 85 /* ARGSUSED */
86 86 int
87 87 on_fault(label_t *ljb)
88 88 { return (0); }
89 89
90 90 void
91 91 no_fault(void)
92 92 {}
93 93
94 94 #else /* __lint */
95 95
96 96 #if defined(__amd64)
97 97
98 98 ENTRY(on_fault)
99 99 movq %gs:CPU_THREAD, %rsi
100 100 leaq catch_fault(%rip), %rdx
101 101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
102 102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
103 103 call smap_disable /* allow user accesses */
104 104 jmp setjmp /* let setjmp do the rest */
105 105
106 106 catch_fault:
107 107 movq %gs:CPU_THREAD, %rsi
108 108 movq T_ONFAULT(%rsi), %rdi /* address of save area */
109 109 xorl %eax, %eax
110 110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
111 111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
112 112 call smap_enable /* disallow user accesses */
113 113 jmp longjmp /* let longjmp do the rest */
114 114 SET_SIZE(on_fault)
115 115
116 116 ENTRY(no_fault)
117 117 movq %gs:CPU_THREAD, %rsi
118 118 xorl %eax, %eax
119 119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
120 120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
121 121 call smap_enable /* disallow user accesses */
122 122 ret
123 123 SET_SIZE(no_fault)
124 124
125 125 #elif defined(__i386)
126 126
127 127 ENTRY(on_fault)
128 128 movl %gs:CPU_THREAD, %edx
129 129 movl 4(%esp), %eax /* jumpbuf address */
130 130 leal catch_fault, %ecx
131 131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
132 132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
133 133 jmp setjmp /* let setjmp do the rest */
134 134
135 135 catch_fault:
136 136 movl %gs:CPU_THREAD, %edx
137 137 xorl %eax, %eax
138 138 movl T_ONFAULT(%edx), %ecx /* address of save area */
139 139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
140 140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
141 141 pushl %ecx
142 142 call longjmp /* let longjmp do the rest */
143 143 SET_SIZE(on_fault)
144 144
145 145 ENTRY(no_fault)
146 146 movl %gs:CPU_THREAD, %edx
147 147 xorl %eax, %eax
148 148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
149 149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
150 150 ret
151 151 SET_SIZE(no_fault)
152 152
153 153 #endif /* __i386 */
154 154 #endif /* __lint */
155 155
156 156 /*
157 157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
158 158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 159 */
160 160
161 161 #if defined(lint)
162 162
163 163 void
164 164 on_trap_trampoline(void)
165 165 {}
166 166
167 167 #else /* __lint */
168 168
169 169 #if defined(__amd64)
170 170
171 171 ENTRY(on_trap_trampoline)
172 172 movq %gs:CPU_THREAD, %rsi
173 173 movq T_ONTRAP(%rsi), %rdi
174 174 addq $OT_JMPBUF, %rdi
175 175 jmp longjmp
176 176 SET_SIZE(on_trap_trampoline)
177 177
178 178 #elif defined(__i386)
179 179
180 180 ENTRY(on_trap_trampoline)
181 181 movl %gs:CPU_THREAD, %eax
182 182 movl T_ONTRAP(%eax), %eax
183 183 addl $OT_JMPBUF, %eax
184 184 pushl %eax
185 185 call longjmp
186 186 SET_SIZE(on_trap_trampoline)
187 187
188 188 #endif /* __i386 */
189 189 #endif /* __lint */
190 190
191 191 /*
192 192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
193 193 * more information about the on_trap() mechanism. If the on_trap_data is the
194 194 * same as the topmost stack element, we just modify that element.
195 195 */
196 196 #if defined(lint)
197 197
198 198 /*ARGSUSED*/
199 199 int
200 200 on_trap(on_trap_data_t *otp, uint_t prot)
201 201 { return (0); }
202 202
203 203 #else /* __lint */
204 204
205 205 #if defined(__amd64)
206 206
207 207 ENTRY(on_trap)
208 208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
209 209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
210 210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
211 211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
212 212 xorl %ecx, %ecx
213 213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
214 214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
215 215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
216 216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
217 217 cmpq %rdi, %rcx /* if (otp == %rcx) */
218 218 je 0f /* don't modify t_ontrap */
219 219
220 220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
221 221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
222 222
223 223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
224 224 jmp setjmp
225 225 SET_SIZE(on_trap)
226 226
227 227 #elif defined(__i386)
228 228
229 229 ENTRY(on_trap)
230 230 movl 4(%esp), %eax /* %eax = otp */
231 231 movl 8(%esp), %edx /* %edx = prot */
232 232
233 233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
234 234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
235 235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
236 236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
237 237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
238 238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
239 239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
240 240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
241 241 cmpl %eax, %ecx /* if (otp == %ecx) */
242 242 je 0f /* don't modify t_ontrap */
243 243
244 244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
245 245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
246 246
247 247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
248 248 movl %eax, 4(%esp) /* put %eax back on the stack */
249 249 jmp setjmp /* let setjmp do the rest */
250 250 SET_SIZE(on_trap)
251 251
252 252 #endif /* __i386 */
253 253 #endif /* __lint */
254 254
255 255 /*
256 256 * Setjmp and longjmp implement non-local gotos using state vectors
257 257 * type label_t.
258 258 */
259 259
260 260 #if defined(__lint)
261 261
262 262 /* ARGSUSED */
263 263 int
264 264 setjmp(label_t *lp)
265 265 { return (0); }
266 266
267 267 /* ARGSUSED */
268 268 void
269 269 longjmp(label_t *lp)
270 270 {}
271 271
272 272 #else /* __lint */
273 273
274 274 #if LABEL_PC != 0
275 275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276 276 #endif /* LABEL_PC != 0 */
277 277
278 278 #if defined(__amd64)
279 279
280 280 ENTRY(setjmp)
281 281 movq %rsp, LABEL_SP(%rdi)
282 282 movq %rbp, LABEL_RBP(%rdi)
283 283 movq %rbx, LABEL_RBX(%rdi)
284 284 movq %r12, LABEL_R12(%rdi)
285 285 movq %r13, LABEL_R13(%rdi)
286 286 movq %r14, LABEL_R14(%rdi)
287 287 movq %r15, LABEL_R15(%rdi)
288 288 movq (%rsp), %rdx /* return address */
289 289 movq %rdx, (%rdi) /* LABEL_PC is 0 */
290 290 xorl %eax, %eax /* return 0 */
291 291 ret
292 292 SET_SIZE(setjmp)
293 293
294 294 ENTRY(longjmp)
295 295 movq LABEL_SP(%rdi), %rsp
296 296 movq LABEL_RBP(%rdi), %rbp
297 297 movq LABEL_RBX(%rdi), %rbx
298 298 movq LABEL_R12(%rdi), %r12
299 299 movq LABEL_R13(%rdi), %r13
300 300 movq LABEL_R14(%rdi), %r14
301 301 movq LABEL_R15(%rdi), %r15
302 302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
303 303 movq %rdx, (%rsp)
304 304 xorl %eax, %eax
305 305 incl %eax /* return 1 */
306 306 ret
307 307 SET_SIZE(longjmp)
308 308
309 309 #elif defined(__i386)
310 310
311 311 ENTRY(setjmp)
312 312 movl 4(%esp), %edx /* address of save area */
313 313 movl %ebp, LABEL_EBP(%edx)
314 314 movl %ebx, LABEL_EBX(%edx)
315 315 movl %esi, LABEL_ESI(%edx)
316 316 movl %edi, LABEL_EDI(%edx)
317 317 movl %esp, 4(%edx)
318 318 movl (%esp), %ecx /* %eip (return address) */
319 319 movl %ecx, (%edx) /* LABEL_PC is 0 */
320 320 subl %eax, %eax /* return 0 */
321 321 ret
322 322 SET_SIZE(setjmp)
323 323
324 324 ENTRY(longjmp)
325 325 movl 4(%esp), %edx /* address of save area */
326 326 movl LABEL_EBP(%edx), %ebp
327 327 movl LABEL_EBX(%edx), %ebx
328 328 movl LABEL_ESI(%edx), %esi
329 329 movl LABEL_EDI(%edx), %edi
330 330 movl 4(%edx), %esp
331 331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
332 332 movl $1, %eax
333 333 addl $4, %esp /* pop ret adr */
334 334 jmp *%ecx /* indirect */
335 335 SET_SIZE(longjmp)
336 336
337 337 #endif /* __i386 */
338 338 #endif /* __lint */
339 339
340 340 /*
341 341 * if a() calls b() calls caller(),
342 342 * caller() returns return address in a().
343 343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 344 * sequence.)
345 345 */
346 346
347 347 #if defined(__lint)
348 348
349 349 caddr_t
350 350 caller(void)
351 351 { return (0); }
352 352
353 353 #else /* __lint */
354 354
355 355 #if defined(__amd64)
356 356
357 357 ENTRY(caller)
358 358 movq 8(%rbp), %rax /* b()'s return pc, in a() */
359 359 ret
360 360 SET_SIZE(caller)
361 361
362 362 #elif defined(__i386)
363 363
364 364 ENTRY(caller)
365 365 movl 4(%ebp), %eax /* b()'s return pc, in a() */
366 366 ret
367 367 SET_SIZE(caller)
368 368
369 369 #endif /* __i386 */
370 370 #endif /* __lint */
371 371
372 372 /*
373 373 * if a() calls callee(), callee() returns the
374 374 * return address in a();
375 375 */
376 376
377 377 #if defined(__lint)
378 378
379 379 caddr_t
380 380 callee(void)
381 381 { return (0); }
382 382
383 383 #else /* __lint */
384 384
385 385 #if defined(__amd64)
386 386
387 387 ENTRY(callee)
388 388 movq (%rsp), %rax /* callee()'s return pc, in a() */
389 389 ret
390 390 SET_SIZE(callee)
391 391
392 392 #elif defined(__i386)
393 393
394 394 ENTRY(callee)
395 395 movl (%esp), %eax /* callee()'s return pc, in a() */
396 396 ret
397 397 SET_SIZE(callee)
398 398
399 399 #endif /* __i386 */
400 400 #endif /* __lint */
401 401
402 402 /*
403 403 * return the current frame pointer
404 404 */
405 405
406 406 #if defined(__lint)
407 407
408 408 greg_t
409 409 getfp(void)
410 410 { return (0); }
411 411
412 412 #else /* __lint */
413 413
414 414 #if defined(__amd64)
415 415
416 416 ENTRY(getfp)
417 417 movq %rbp, %rax
418 418 ret
419 419 SET_SIZE(getfp)
420 420
421 421 #elif defined(__i386)
422 422
423 423 ENTRY(getfp)
424 424 movl %ebp, %eax
425 425 ret
426 426 SET_SIZE(getfp)
427 427
428 428 #endif /* __i386 */
|
↓ open down ↓ |
392 lines elided |
↑ open up ↑ |
429 429 #endif /* __lint */
430 430
431 431 /*
432 432 * Invalidate a single page table entry in the TLB
433 433 */
434 434
435 435 #if defined(__lint)
436 436
437 437 /* ARGSUSED */
438 438 void
439 -mmu_tlbflush_entry(caddr_t m)
439 +mmu_invlpg(caddr_t m)
440 440 {}
441 441
442 442 #else /* __lint */
443 443
444 -#if defined(__amd64)
445 -
446 - ENTRY(mmu_tlbflush_entry)
444 + ENTRY(mmu_invlpg)
447 445 invlpg (%rdi)
448 446 ret
449 - SET_SIZE(mmu_tlbflush_entry)
447 + SET_SIZE(mmu_invlpg)
450 448
451 -#elif defined(__i386)
452 -
453 - ENTRY(mmu_tlbflush_entry)
454 - movl 4(%esp), %eax
455 - invlpg (%eax)
456 - ret
457 - SET_SIZE(mmu_tlbflush_entry)
458 -
459 -#endif /* __i386 */
460 449 #endif /* __lint */
461 450
462 451
463 452 /*
464 453 * Get/Set the value of various control registers
465 454 */
466 455
467 456 #if defined(__lint)
468 457
469 458 ulong_t
470 459 getcr0(void)
471 460 { return (0); }
472 461
473 462 /* ARGSUSED */
474 463 void
475 464 setcr0(ulong_t value)
476 465 {}
477 466
478 467 ulong_t
479 468 getcr2(void)
480 469 { return (0); }
481 470
482 471 ulong_t
483 472 getcr3(void)
484 473 { return (0); }
485 474
486 475 #if !defined(__xpv)
487 476 /* ARGSUSED */
488 477 void
489 478 setcr3(ulong_t val)
490 479 {}
491 480
492 481 void
493 482 reload_cr3(void)
494 483 {}
495 484 #endif
496 485
497 486 ulong_t
498 487 getcr4(void)
499 488 { return (0); }
500 489
501 490 /* ARGSUSED */
502 491 void
503 492 setcr4(ulong_t val)
504 493 {}
505 494
506 495 #if defined(__amd64)
507 496
508 497 ulong_t
509 498 getcr8(void)
510 499 { return (0); }
511 500
512 501 /* ARGSUSED */
513 502 void
514 503 setcr8(ulong_t val)
515 504 {}
516 505
517 506 #endif /* __amd64 */
518 507
519 508 #else /* __lint */
520 509
521 510 #if defined(__amd64)
522 511
523 512 ENTRY(getcr0)
524 513 movq %cr0, %rax
525 514 ret
526 515 SET_SIZE(getcr0)
527 516
528 517 ENTRY(setcr0)
529 518 movq %rdi, %cr0
530 519 ret
531 520 SET_SIZE(setcr0)
532 521
533 522 ENTRY(getcr2)
534 523 #if defined(__xpv)
535 524 movq %gs:CPU_VCPU_INFO, %rax
536 525 movq VCPU_INFO_ARCH_CR2(%rax), %rax
537 526 #else
538 527 movq %cr2, %rax
539 528 #endif
540 529 ret
541 530 SET_SIZE(getcr2)
542 531
543 532 ENTRY(getcr3)
544 533 movq %cr3, %rax
545 534 ret
546 535 SET_SIZE(getcr3)
547 536
548 537 #if !defined(__xpv)
549 538
550 539 ENTRY(setcr3)
551 540 movq %rdi, %cr3
552 541 ret
553 542 SET_SIZE(setcr3)
554 543
555 544 ENTRY(reload_cr3)
556 545 movq %cr3, %rdi
557 546 movq %rdi, %cr3
558 547 ret
559 548 SET_SIZE(reload_cr3)
560 549
561 550 #endif /* __xpv */
562 551
563 552 ENTRY(getcr4)
564 553 movq %cr4, %rax
565 554 ret
566 555 SET_SIZE(getcr4)
567 556
568 557 ENTRY(setcr4)
569 558 movq %rdi, %cr4
570 559 ret
571 560 SET_SIZE(setcr4)
572 561
573 562 ENTRY(getcr8)
574 563 movq %cr8, %rax
575 564 ret
576 565 SET_SIZE(getcr8)
577 566
578 567 ENTRY(setcr8)
579 568 movq %rdi, %cr8
580 569 ret
581 570 SET_SIZE(setcr8)
582 571
583 572 #elif defined(__i386)
584 573
585 574 ENTRY(getcr0)
586 575 movl %cr0, %eax
587 576 ret
588 577 SET_SIZE(getcr0)
589 578
590 579 ENTRY(setcr0)
591 580 movl 4(%esp), %eax
592 581 movl %eax, %cr0
593 582 ret
594 583 SET_SIZE(setcr0)
595 584
596 585 /*
597 586 * "lock mov %cr0" is used on processors which indicate it is
598 587 * supported via CPUID. Normally the 32 bit TPR is accessed via
599 588 * the local APIC.
600 589 */
601 590 ENTRY(getcr8)
602 591 lock
603 592 movl %cr0, %eax
604 593 ret
605 594 SET_SIZE(getcr8)
606 595
607 596 ENTRY(setcr8)
608 597 movl 4(%esp), %eax
609 598 lock
610 599 movl %eax, %cr0
611 600 ret
612 601 SET_SIZE(setcr8)
613 602
614 603 ENTRY(getcr2)
615 604 #if defined(__xpv)
616 605 movl %gs:CPU_VCPU_INFO, %eax
617 606 movl VCPU_INFO_ARCH_CR2(%eax), %eax
618 607 #else
619 608 movl %cr2, %eax
620 609 #endif
621 610 ret
622 611 SET_SIZE(getcr2)
623 612
624 613 ENTRY(getcr3)
625 614 movl %cr3, %eax
626 615 ret
627 616 SET_SIZE(getcr3)
628 617
629 618 #if !defined(__xpv)
630 619
631 620 ENTRY(setcr3)
632 621 movl 4(%esp), %eax
633 622 movl %eax, %cr3
634 623 ret
635 624 SET_SIZE(setcr3)
636 625
637 626 ENTRY(reload_cr3)
638 627 movl %cr3, %eax
639 628 movl %eax, %cr3
640 629 ret
641 630 SET_SIZE(reload_cr3)
642 631
643 632 #endif /* __xpv */
644 633
645 634 ENTRY(getcr4)
646 635 movl %cr4, %eax
647 636 ret
648 637 SET_SIZE(getcr4)
649 638
650 639 ENTRY(setcr4)
651 640 movl 4(%esp), %eax
652 641 movl %eax, %cr4
653 642 ret
654 643 SET_SIZE(setcr4)
655 644
656 645 #endif /* __i386 */
657 646 #endif /* __lint */
658 647
659 648 #if defined(__lint)
660 649
661 650 /*ARGSUSED*/
662 651 uint32_t
663 652 __cpuid_insn(struct cpuid_regs *regs)
664 653 { return (0); }
665 654
666 655 #else /* __lint */
667 656
668 657 #if defined(__amd64)
669 658
670 659 ENTRY(__cpuid_insn)
671 660 movq %rbx, %r8
672 661 movq %rcx, %r9
673 662 movq %rdx, %r11
674 663 movl (%rdi), %eax /* %eax = regs->cp_eax */
675 664 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
676 665 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
677 666 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
678 667 cpuid
679 668 movl %eax, (%rdi) /* regs->cp_eax = %eax */
680 669 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
681 670 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
682 671 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
683 672 movq %r8, %rbx
684 673 movq %r9, %rcx
685 674 movq %r11, %rdx
686 675 ret
687 676 SET_SIZE(__cpuid_insn)
688 677
689 678 #elif defined(__i386)
690 679
691 680 ENTRY(__cpuid_insn)
692 681 pushl %ebp
693 682 movl 0x8(%esp), %ebp /* %ebp = regs */
694 683 pushl %ebx
695 684 pushl %ecx
696 685 pushl %edx
697 686 movl (%ebp), %eax /* %eax = regs->cp_eax */
698 687 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
699 688 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
700 689 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
701 690 cpuid
702 691 movl %eax, (%ebp) /* regs->cp_eax = %eax */
703 692 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
704 693 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
705 694 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
706 695 popl %edx
707 696 popl %ecx
708 697 popl %ebx
709 698 popl %ebp
710 699 ret
711 700 SET_SIZE(__cpuid_insn)
712 701
713 702 #endif /* __i386 */
714 703 #endif /* __lint */
715 704
716 705 #if defined(__lint)
717 706
718 707 /*ARGSUSED*/
719 708 void
720 709 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
721 710 {}
722 711
723 712 #else /* __lint */
724 713
725 714 #if defined(__amd64)
726 715
727 716 ENTRY_NP(i86_monitor)
728 717 pushq %rbp
729 718 movq %rsp, %rbp
730 719 movq %rdi, %rax /* addr */
731 720 movq %rsi, %rcx /* extensions */
732 721 /* rdx contains input arg3: hints */
733 722 clflush (%rax)
734 723 .byte 0x0f, 0x01, 0xc8 /* monitor */
735 724 leave
736 725 ret
737 726 SET_SIZE(i86_monitor)
738 727
739 728 #elif defined(__i386)
740 729
741 730 ENTRY_NP(i86_monitor)
742 731 pushl %ebp
743 732 movl %esp, %ebp
744 733 movl 0x8(%ebp),%eax /* addr */
745 734 movl 0xc(%ebp),%ecx /* extensions */
746 735 movl 0x10(%ebp),%edx /* hints */
747 736 clflush (%eax)
748 737 .byte 0x0f, 0x01, 0xc8 /* monitor */
749 738 leave
750 739 ret
751 740 SET_SIZE(i86_monitor)
752 741
753 742 #endif /* __i386 */
754 743 #endif /* __lint */
755 744
756 745 #if defined(__lint)
757 746
758 747 /*ARGSUSED*/
759 748 void
760 749 i86_mwait(uint32_t data, uint32_t extensions)
761 750 {}
762 751
763 752 #else /* __lint */
764 753
765 754 #if defined(__amd64)
766 755
767 756 ENTRY_NP(i86_mwait)
768 757 pushq %rbp
769 758 movq %rsp, %rbp
770 759 movq %rdi, %rax /* data */
771 760 movq %rsi, %rcx /* extensions */
772 761 .byte 0x0f, 0x01, 0xc9 /* mwait */
773 762 leave
774 763 ret
775 764 SET_SIZE(i86_mwait)
776 765
777 766 #elif defined(__i386)
778 767
779 768 ENTRY_NP(i86_mwait)
780 769 pushl %ebp
781 770 movl %esp, %ebp
782 771 movl 0x8(%ebp),%eax /* data */
783 772 movl 0xc(%ebp),%ecx /* extensions */
784 773 .byte 0x0f, 0x01, 0xc9 /* mwait */
785 774 leave
786 775 ret
787 776 SET_SIZE(i86_mwait)
788 777
789 778 #endif /* __i386 */
790 779 #endif /* __lint */
791 780
792 781 #if defined(__xpv)
793 782 /*
794 783 * Defined in C
795 784 */
796 785 #else
797 786
798 787 #if defined(__lint)
799 788
800 789 hrtime_t
801 790 tsc_read(void)
802 791 {
803 792 return (0);
804 793 }
805 794
806 795 #else /* __lint */
807 796
808 797 #if defined(__amd64)
809 798
810 799 ENTRY_NP(tsc_read)
811 800 movq %rbx, %r11
812 801 movl $0, %eax
813 802 cpuid
814 803 rdtsc
815 804 movq %r11, %rbx
816 805 shlq $32, %rdx
817 806 orq %rdx, %rax
818 807 ret
819 808 .globl _tsc_mfence_start
820 809 _tsc_mfence_start:
821 810 mfence
822 811 rdtsc
823 812 shlq $32, %rdx
824 813 orq %rdx, %rax
825 814 ret
826 815 .globl _tsc_mfence_end
827 816 _tsc_mfence_end:
828 817 .globl _tscp_start
829 818 _tscp_start:
830 819 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
831 820 shlq $32, %rdx
832 821 orq %rdx, %rax
833 822 ret
834 823 .globl _tscp_end
835 824 _tscp_end:
836 825 .globl _no_rdtsc_start
837 826 _no_rdtsc_start:
838 827 xorl %edx, %edx
839 828 xorl %eax, %eax
840 829 ret
841 830 .globl _no_rdtsc_end
842 831 _no_rdtsc_end:
843 832 .globl _tsc_lfence_start
844 833 _tsc_lfence_start:
845 834 lfence
846 835 rdtsc
847 836 shlq $32, %rdx
848 837 orq %rdx, %rax
849 838 ret
850 839 .globl _tsc_lfence_end
851 840 _tsc_lfence_end:
852 841 SET_SIZE(tsc_read)
853 842
854 843 #else /* __i386 */
855 844
856 845 ENTRY_NP(tsc_read)
857 846 pushl %ebx
858 847 movl $0, %eax
859 848 cpuid
860 849 rdtsc
861 850 popl %ebx
862 851 ret
863 852 .globl _tsc_mfence_start
864 853 _tsc_mfence_start:
865 854 mfence
866 855 rdtsc
867 856 ret
868 857 .globl _tsc_mfence_end
869 858 _tsc_mfence_end:
870 859 .globl _tscp_start
871 860 _tscp_start:
872 861 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
873 862 ret
874 863 .globl _tscp_end
875 864 _tscp_end:
876 865 .globl _no_rdtsc_start
877 866 _no_rdtsc_start:
878 867 xorl %edx, %edx
879 868 xorl %eax, %eax
880 869 ret
881 870 .globl _no_rdtsc_end
882 871 _no_rdtsc_end:
883 872 .globl _tsc_lfence_start
884 873 _tsc_lfence_start:
885 874 lfence
886 875 rdtsc
887 876 ret
888 877 .globl _tsc_lfence_end
889 878 _tsc_lfence_end:
890 879 SET_SIZE(tsc_read)
891 880
892 881 #endif /* __i386 */
893 882
894 883 #endif /* __lint */
895 884
896 885
897 886 #endif /* __xpv */
898 887
899 888 #ifdef __lint
900 889 /*
901 890 * Do not use this function for obtaining clock tick. This
902 891 * is called by callers who do not need to have a guarenteed
903 892 * correct tick value. The proper routine to use is tsc_read().
904 893 */
905 894 u_longlong_t
906 895 randtick(void)
907 896 {
908 897 return (0);
909 898 }
910 899 #else
911 900 #if defined(__amd64)
912 901 ENTRY_NP(randtick)
913 902 rdtsc
914 903 shlq $32, %rdx
915 904 orq %rdx, %rax
916 905 ret
917 906 SET_SIZE(randtick)
918 907 #else
919 908 ENTRY_NP(randtick)
920 909 rdtsc
921 910 ret
922 911 SET_SIZE(randtick)
923 912 #endif /* __i386 */
924 913 #endif /* __lint */
925 914 /*
926 915 * Insert entryp after predp in a doubly linked list.
927 916 */
928 917
929 918 #if defined(__lint)
930 919
931 920 /*ARGSUSED*/
932 921 void
933 922 _insque(caddr_t entryp, caddr_t predp)
934 923 {}
935 924
936 925 #else /* __lint */
937 926
938 927 #if defined(__amd64)
939 928
940 929 ENTRY(_insque)
941 930 movq (%rsi), %rax /* predp->forw */
942 931 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
943 932 movq %rax, (%rdi) /* entryp->forw = predp->forw */
944 933 movq %rdi, (%rsi) /* predp->forw = entryp */
945 934 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
946 935 ret
947 936 SET_SIZE(_insque)
948 937
949 938 #elif defined(__i386)
950 939
951 940 ENTRY(_insque)
952 941 movl 8(%esp), %edx
953 942 movl 4(%esp), %ecx
954 943 movl (%edx), %eax /* predp->forw */
955 944 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
956 945 movl %eax, (%ecx) /* entryp->forw = predp->forw */
957 946 movl %ecx, (%edx) /* predp->forw = entryp */
958 947 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
959 948 ret
960 949 SET_SIZE(_insque)
961 950
962 951 #endif /* __i386 */
963 952 #endif /* __lint */
964 953
965 954 /*
966 955 * Remove entryp from a doubly linked list
967 956 */
968 957
969 958 #if defined(__lint)
970 959
971 960 /*ARGSUSED*/
972 961 void
973 962 _remque(caddr_t entryp)
974 963 {}
975 964
976 965 #else /* __lint */
977 966
978 967 #if defined(__amd64)
979 968
980 969 ENTRY(_remque)
981 970 movq (%rdi), %rax /* entry->forw */
982 971 movq CPTRSIZE(%rdi), %rdx /* entry->back */
983 972 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
984 973 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
985 974 ret
986 975 SET_SIZE(_remque)
987 976
988 977 #elif defined(__i386)
989 978
990 979 ENTRY(_remque)
991 980 movl 4(%esp), %ecx
992 981 movl (%ecx), %eax /* entry->forw */
993 982 movl CPTRSIZE(%ecx), %edx /* entry->back */
994 983 movl %eax, (%edx) /* entry->back->forw = entry->forw */
995 984 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
996 985 ret
997 986 SET_SIZE(_remque)
998 987
999 988 #endif /* __i386 */
1000 989 #endif /* __lint */
1001 990
1002 991 /*
1003 992 * Returns the number of
1004 993 * non-NULL bytes in string argument.
1005 994 */
1006 995
1007 996 #if defined(__lint)
1008 997
1009 998 /* ARGSUSED */
1010 999 size_t
1011 1000 strlen(const char *str)
1012 1001 { return (0); }
1013 1002
1014 1003 #else /* __lint */
1015 1004
1016 1005 #if defined(__amd64)
1017 1006
1018 1007 /*
1019 1008 * This is close to a simple transliteration of a C version of this
1020 1009 * routine. We should either just -make- this be a C version, or
1021 1010 * justify having it in assembler by making it significantly faster.
1022 1011 *
1023 1012 * size_t
1024 1013 * strlen(const char *s)
1025 1014 * {
1026 1015 * const char *s0;
1027 1016 * #if defined(DEBUG)
1028 1017 * if ((uintptr_t)s < KERNELBASE)
1029 1018 * panic(.str_panic_msg);
1030 1019 * #endif
1031 1020 * for (s0 = s; *s; s++)
1032 1021 * ;
1033 1022 * return (s - s0);
1034 1023 * }
1035 1024 */
1036 1025
1037 1026 ENTRY(strlen)
1038 1027 #ifdef DEBUG
1039 1028 movq postbootkernelbase(%rip), %rax
1040 1029 cmpq %rax, %rdi
1041 1030 jae str_valid
1042 1031 pushq %rbp
1043 1032 movq %rsp, %rbp
1044 1033 leaq .str_panic_msg(%rip), %rdi
1045 1034 xorl %eax, %eax
1046 1035 call panic
1047 1036 #endif /* DEBUG */
1048 1037 str_valid:
1049 1038 cmpb $0, (%rdi)
1050 1039 movq %rdi, %rax
1051 1040 je .null_found
1052 1041 .align 4
1053 1042 .strlen_loop:
1054 1043 incq %rdi
1055 1044 cmpb $0, (%rdi)
1056 1045 jne .strlen_loop
1057 1046 .null_found:
1058 1047 subq %rax, %rdi
1059 1048 movq %rdi, %rax
1060 1049 ret
1061 1050 SET_SIZE(strlen)
1062 1051
1063 1052 #elif defined(__i386)
1064 1053
1065 1054 ENTRY(strlen)
1066 1055 #ifdef DEBUG
1067 1056 movl postbootkernelbase, %eax
1068 1057 cmpl %eax, 4(%esp)
1069 1058 jae str_valid
1070 1059 pushl %ebp
1071 1060 movl %esp, %ebp
1072 1061 pushl $.str_panic_msg
1073 1062 call panic
1074 1063 #endif /* DEBUG */
1075 1064
1076 1065 str_valid:
1077 1066 movl 4(%esp), %eax /* %eax = string address */
1078 1067 testl $3, %eax /* if %eax not word aligned */
1079 1068 jnz .not_word_aligned /* goto .not_word_aligned */
1080 1069 .align 4
1081 1070 .word_aligned:
1082 1071 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1083 1072 movl $0x7f7f7f7f, %ecx
1084 1073 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1085 1074 addl $4, %eax /* next word */
1086 1075 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1087 1076 orl %edx, %ecx /* %ecx |= %edx */
1088 1077 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1089 1078 cmpl $0x80808080, %ecx /* if no null byte in this word */
1090 1079 je .word_aligned /* goto .word_aligned */
1091 1080 subl $4, %eax /* post-incremented */
1092 1081 .not_word_aligned:
1093 1082 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1094 1083 je .null_found /* goto .null_found */
1095 1084 incl %eax /* next byte */
1096 1085 testl $3, %eax /* if %eax not word aligned */
1097 1086 jnz .not_word_aligned /* goto .not_word_aligned */
1098 1087 jmp .word_aligned /* goto .word_aligned */
1099 1088 .align 4
1100 1089 .null_found:
1101 1090 subl 4(%esp), %eax /* %eax -= string address */
1102 1091 ret
1103 1092 SET_SIZE(strlen)
1104 1093
1105 1094 #endif /* __i386 */
1106 1095
1107 1096 #ifdef DEBUG
1108 1097 .text
1109 1098 .str_panic_msg:
1110 1099 .string "strlen: argument below kernelbase"
1111 1100 #endif /* DEBUG */
1112 1101
1113 1102 #endif /* __lint */
1114 1103
1115 1104 /*
1116 1105 * Berkeley 4.3 introduced symbolically named interrupt levels
1117 1106 * as a way deal with priority in a machine independent fashion.
1118 1107 * Numbered priorities are machine specific, and should be
1119 1108 * discouraged where possible.
1120 1109 *
1121 1110 * Note, for the machine specific priorities there are
1122 1111 * examples listed for devices that use a particular priority.
1123 1112 * It should not be construed that all devices of that
1124 1113 * type should be at that priority. It is currently were
1125 1114 * the current devices fit into the priority scheme based
1126 1115 * upon time criticalness.
1127 1116 *
1128 1117 * The underlying assumption of these assignments is that
1129 1118 * IPL 10 is the highest level from which a device
1130 1119 * routine can call wakeup. Devices that interrupt from higher
1131 1120 * levels are restricted in what they can do. If they need
1132 1121 * kernels services they should schedule a routine at a lower
1133 1122 * level (via software interrupt) to do the required
1134 1123 * processing.
1135 1124 *
1136 1125 * Examples of this higher usage:
1137 1126 * Level Usage
1138 1127 * 14 Profiling clock (and PROM uart polling clock)
1139 1128 * 12 Serial ports
1140 1129 *
1141 1130 * The serial ports request lower level processing on level 6.
1142 1131 *
1143 1132 * Also, almost all splN routines (where N is a number or a
1144 1133 * mnemonic) will do a RAISE(), on the assumption that they are
1145 1134 * never used to lower our priority.
1146 1135 * The exceptions are:
1147 1136 * spl8() Because you can't be above 15 to begin with!
1148 1137 * splzs() Because this is used at boot time to lower our
1149 1138 * priority, to allow the PROM to poll the uart.
1150 1139 * spl0() Used to lower priority to 0.
1151 1140 */
1152 1141
1153 1142 #if defined(__lint)
1154 1143
1155 1144 int spl0(void) { return (0); }
1156 1145 int spl6(void) { return (0); }
1157 1146 int spl7(void) { return (0); }
1158 1147 int spl8(void) { return (0); }
1159 1148 int splhigh(void) { return (0); }
1160 1149 int splhi(void) { return (0); }
1161 1150 int splzs(void) { return (0); }
1162 1151
1163 1152 /* ARGSUSED */
1164 1153 void
1165 1154 splx(int level)
1166 1155 {}
1167 1156
1168 1157 #else /* __lint */
1169 1158
1170 1159 #if defined(__amd64)
1171 1160
1172 1161 #define SETPRI(level) \
1173 1162 movl $/**/level, %edi; /* new priority */ \
1174 1163 jmp do_splx /* redirect to do_splx */
1175 1164
1176 1165 #define RAISE(level) \
1177 1166 movl $/**/level, %edi; /* new priority */ \
1178 1167 jmp splr /* redirect to splr */
1179 1168
1180 1169 #elif defined(__i386)
1181 1170
1182 1171 #define SETPRI(level) \
1183 1172 pushl $/**/level; /* new priority */ \
1184 1173 call do_splx; /* invoke common splx code */ \
1185 1174 addl $4, %esp; /* unstack arg */ \
1186 1175 ret
1187 1176
1188 1177 #define RAISE(level) \
1189 1178 pushl $/**/level; /* new priority */ \
1190 1179 call splr; /* invoke common splr code */ \
1191 1180 addl $4, %esp; /* unstack args */ \
1192 1181 ret
1193 1182
1194 1183 #endif /* __i386 */
1195 1184
1196 1185 /* locks out all interrupts, including memory errors */
1197 1186 ENTRY(spl8)
1198 1187 SETPRI(15)
1199 1188 SET_SIZE(spl8)
1200 1189
1201 1190 /* just below the level that profiling runs */
1202 1191 ENTRY(spl7)
1203 1192 RAISE(13)
1204 1193 SET_SIZE(spl7)
1205 1194
1206 1195 /* sun specific - highest priority onboard serial i/o asy ports */
1207 1196 ENTRY(splzs)
1208 1197 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1209 1198 SET_SIZE(splzs)
1210 1199
1211 1200 ENTRY(splhi)
1212 1201 ALTENTRY(splhigh)
1213 1202 ALTENTRY(spl6)
1214 1203 ALTENTRY(i_ddi_splhigh)
1215 1204
1216 1205 RAISE(DISP_LEVEL)
1217 1206
1218 1207 SET_SIZE(i_ddi_splhigh)
1219 1208 SET_SIZE(spl6)
1220 1209 SET_SIZE(splhigh)
1221 1210 SET_SIZE(splhi)
1222 1211
1223 1212 /* allow all interrupts */
1224 1213 ENTRY(spl0)
1225 1214 SETPRI(0)
1226 1215 SET_SIZE(spl0)
1227 1216
1228 1217
1229 1218 /* splx implementation */
1230 1219 ENTRY(splx)
1231 1220 jmp do_splx /* redirect to common splx code */
1232 1221 SET_SIZE(splx)
1233 1222
1234 1223 #endif /* __lint */
1235 1224
1236 1225 #if defined(__i386)
1237 1226
1238 1227 /*
1239 1228 * Read and write the %gs register
1240 1229 */
1241 1230
1242 1231 #if defined(__lint)
1243 1232
1244 1233 /*ARGSUSED*/
1245 1234 uint16_t
1246 1235 getgs(void)
1247 1236 { return (0); }
1248 1237
1249 1238 /*ARGSUSED*/
1250 1239 void
1251 1240 setgs(uint16_t sel)
1252 1241 {}
1253 1242
1254 1243 #else /* __lint */
1255 1244
1256 1245 ENTRY(getgs)
1257 1246 clr %eax
1258 1247 movw %gs, %ax
1259 1248 ret
1260 1249 SET_SIZE(getgs)
1261 1250
1262 1251 ENTRY(setgs)
1263 1252 movw 4(%esp), %gs
1264 1253 ret
1265 1254 SET_SIZE(setgs)
1266 1255
1267 1256 #endif /* __lint */
1268 1257 #endif /* __i386 */
1269 1258
1270 1259 #if defined(__lint)
1271 1260
1272 1261 void
1273 1262 pc_reset(void)
1274 1263 {}
1275 1264
1276 1265 void
1277 1266 efi_reset(void)
1278 1267 {}
1279 1268
1280 1269 #else /* __lint */
1281 1270
1282 1271 ENTRY(wait_500ms)
1283 1272 #if defined(__amd64)
1284 1273 pushq %rbx
1285 1274 #elif defined(__i386)
1286 1275 push %ebx
1287 1276 #endif
1288 1277 movl $50000, %ebx
1289 1278 1:
1290 1279 call tenmicrosec
1291 1280 decl %ebx
1292 1281 jnz 1b
1293 1282 #if defined(__amd64)
1294 1283 popq %rbx
1295 1284 #elif defined(__i386)
1296 1285 pop %ebx
1297 1286 #endif
1298 1287 ret
1299 1288 SET_SIZE(wait_500ms)
1300 1289
1301 1290 #define RESET_METHOD_KBC 1
1302 1291 #define RESET_METHOD_PORT92 2
1303 1292 #define RESET_METHOD_PCI 4
1304 1293
1305 1294 DGDEF3(pc_reset_methods, 4, 8)
1306 1295 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1307 1296
1308 1297 ENTRY(pc_reset)
1309 1298
1310 1299 #if defined(__i386)
1311 1300 testl $RESET_METHOD_KBC, pc_reset_methods
1312 1301 #elif defined(__amd64)
1313 1302 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1314 1303 #endif
1315 1304 jz 1f
1316 1305
1317 1306 /
1318 1307 / Try the classic keyboard controller-triggered reset.
1319 1308 /
1320 1309 movw $0x64, %dx
1321 1310 movb $0xfe, %al
1322 1311 outb (%dx)
1323 1312
1324 1313 / Wait up to 500 milliseconds here for the keyboard controller
1325 1314 / to pull the reset line. On some systems where the keyboard
1326 1315 / controller is slow to pull the reset line, the next reset method
1327 1316 / may be executed (which may be bad if those systems hang when the
1328 1317 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1329 1318 / and Ferrari 4000 (doesn't like the cf9 reset method))
1330 1319
1331 1320 call wait_500ms
1332 1321
1333 1322 1:
1334 1323 #if defined(__i386)
1335 1324 testl $RESET_METHOD_PORT92, pc_reset_methods
1336 1325 #elif defined(__amd64)
1337 1326 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1338 1327 #endif
1339 1328 jz 3f
1340 1329
1341 1330 /
1342 1331 / Try port 0x92 fast reset
1343 1332 /
1344 1333 movw $0x92, %dx
1345 1334 inb (%dx)
1346 1335 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1347 1336 je 1f
1348 1337 testb $1, %al / If bit 0
1349 1338 jz 2f / is clear, jump to perform the reset
1350 1339 andb $0xfe, %al / otherwise,
1351 1340 outb (%dx) / clear bit 0 first, then
1352 1341 2:
1353 1342 orb $1, %al / Set bit 0
1354 1343 outb (%dx) / and reset the system
1355 1344 1:
1356 1345
1357 1346 call wait_500ms
1358 1347
1359 1348 3:
1360 1349 #if defined(__i386)
1361 1350 testl $RESET_METHOD_PCI, pc_reset_methods
1362 1351 #elif defined(__amd64)
1363 1352 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1364 1353 #endif
1365 1354 jz 4f
1366 1355
1367 1356 / Try the PCI (soft) reset vector (should work on all modern systems,
1368 1357 / but has been shown to cause problems on 450NX systems, and some newer
1369 1358 / systems (e.g. ATI IXP400-equipped systems))
1370 1359 / When resetting via this method, 2 writes are required. The first
1371 1360 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1372 1361 / power cycle).
1373 1362 / The reset occurs on the second write, during bit 2's transition from
1374 1363 / 0->1.
1375 1364 movw $0xcf9, %dx
1376 1365 movb $0x2, %al / Reset mode = hard, no power cycle
1377 1366 outb (%dx)
1378 1367 movb $0x6, %al
1379 1368 outb (%dx)
1380 1369
1381 1370 call wait_500ms
1382 1371
1383 1372 4:
1384 1373 /
1385 1374 / port 0xcf9 failed also. Last-ditch effort is to
1386 1375 / triple-fault the CPU.
1387 1376 / Also, use triple fault for EFI firmware
1388 1377 /
1389 1378 ENTRY(efi_reset)
1390 1379 #if defined(__amd64)
1391 1380 pushq $0x0
1392 1381 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1393 1382 lidt (%rsp)
1394 1383 #elif defined(__i386)
1395 1384 pushl $0x0
1396 1385 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1397 1386 lidt (%esp)
1398 1387 #endif
1399 1388 int $0x0 / Trigger interrupt, generate triple-fault
1400 1389
1401 1390 cli
1402 1391 hlt / Wait forever
1403 1392 /*NOTREACHED*/
1404 1393 SET_SIZE(efi_reset)
1405 1394 SET_SIZE(pc_reset)
1406 1395
1407 1396 #endif /* __lint */
1408 1397
1409 1398 /*
1410 1399 * C callable in and out routines
1411 1400 */
1412 1401
1413 1402 #if defined(__lint)
1414 1403
1415 1404 /* ARGSUSED */
1416 1405 void
1417 1406 outl(int port_address, uint32_t val)
1418 1407 {}
1419 1408
1420 1409 #else /* __lint */
1421 1410
1422 1411 #if defined(__amd64)
1423 1412
1424 1413 ENTRY(outl)
1425 1414 movw %di, %dx
1426 1415 movl %esi, %eax
1427 1416 outl (%dx)
1428 1417 ret
1429 1418 SET_SIZE(outl)
1430 1419
1431 1420 #elif defined(__i386)
1432 1421
1433 1422 .set PORT, 4
1434 1423 .set VAL, 8
1435 1424
1436 1425 ENTRY(outl)
1437 1426 movw PORT(%esp), %dx
1438 1427 movl VAL(%esp), %eax
1439 1428 outl (%dx)
1440 1429 ret
1441 1430 SET_SIZE(outl)
1442 1431
1443 1432 #endif /* __i386 */
1444 1433 #endif /* __lint */
1445 1434
1446 1435 #if defined(__lint)
1447 1436
1448 1437 /* ARGSUSED */
1449 1438 void
1450 1439 outw(int port_address, uint16_t val)
1451 1440 {}
1452 1441
1453 1442 #else /* __lint */
1454 1443
1455 1444 #if defined(__amd64)
1456 1445
1457 1446 ENTRY(outw)
1458 1447 movw %di, %dx
1459 1448 movw %si, %ax
1460 1449 D16 outl (%dx) /* XX64 why not outw? */
1461 1450 ret
1462 1451 SET_SIZE(outw)
1463 1452
1464 1453 #elif defined(__i386)
1465 1454
1466 1455 ENTRY(outw)
1467 1456 movw PORT(%esp), %dx
1468 1457 movw VAL(%esp), %ax
1469 1458 D16 outl (%dx)
1470 1459 ret
1471 1460 SET_SIZE(outw)
1472 1461
1473 1462 #endif /* __i386 */
1474 1463 #endif /* __lint */
1475 1464
1476 1465 #if defined(__lint)
1477 1466
1478 1467 /* ARGSUSED */
1479 1468 void
1480 1469 outb(int port_address, uint8_t val)
1481 1470 {}
1482 1471
1483 1472 #else /* __lint */
1484 1473
1485 1474 #if defined(__amd64)
1486 1475
1487 1476 ENTRY(outb)
1488 1477 movw %di, %dx
1489 1478 movb %sil, %al
1490 1479 outb (%dx)
1491 1480 ret
1492 1481 SET_SIZE(outb)
1493 1482
1494 1483 #elif defined(__i386)
1495 1484
1496 1485 ENTRY(outb)
1497 1486 movw PORT(%esp), %dx
1498 1487 movb VAL(%esp), %al
1499 1488 outb (%dx)
1500 1489 ret
1501 1490 SET_SIZE(outb)
1502 1491
1503 1492 #endif /* __i386 */
1504 1493 #endif /* __lint */
1505 1494
1506 1495 #if defined(__lint)
1507 1496
1508 1497 /* ARGSUSED */
1509 1498 uint32_t
1510 1499 inl(int port_address)
1511 1500 { return (0); }
1512 1501
1513 1502 #else /* __lint */
1514 1503
1515 1504 #if defined(__amd64)
1516 1505
1517 1506 ENTRY(inl)
1518 1507 xorl %eax, %eax
1519 1508 movw %di, %dx
1520 1509 inl (%dx)
1521 1510 ret
1522 1511 SET_SIZE(inl)
1523 1512
1524 1513 #elif defined(__i386)
1525 1514
1526 1515 ENTRY(inl)
1527 1516 movw PORT(%esp), %dx
1528 1517 inl (%dx)
1529 1518 ret
1530 1519 SET_SIZE(inl)
1531 1520
1532 1521 #endif /* __i386 */
1533 1522 #endif /* __lint */
1534 1523
1535 1524 #if defined(__lint)
1536 1525
1537 1526 /* ARGSUSED */
1538 1527 uint16_t
1539 1528 inw(int port_address)
1540 1529 { return (0); }
1541 1530
1542 1531 #else /* __lint */
1543 1532
1544 1533 #if defined(__amd64)
1545 1534
1546 1535 ENTRY(inw)
1547 1536 xorl %eax, %eax
1548 1537 movw %di, %dx
1549 1538 D16 inl (%dx)
1550 1539 ret
1551 1540 SET_SIZE(inw)
1552 1541
1553 1542 #elif defined(__i386)
1554 1543
1555 1544 ENTRY(inw)
1556 1545 subl %eax, %eax
1557 1546 movw PORT(%esp), %dx
1558 1547 D16 inl (%dx)
1559 1548 ret
1560 1549 SET_SIZE(inw)
1561 1550
1562 1551 #endif /* __i386 */
1563 1552 #endif /* __lint */
1564 1553
1565 1554
1566 1555 #if defined(__lint)
1567 1556
1568 1557 /* ARGSUSED */
1569 1558 uint8_t
1570 1559 inb(int port_address)
1571 1560 { return (0); }
1572 1561
1573 1562 #else /* __lint */
1574 1563
1575 1564 #if defined(__amd64)
1576 1565
1577 1566 ENTRY(inb)
1578 1567 xorl %eax, %eax
1579 1568 movw %di, %dx
1580 1569 inb (%dx)
1581 1570 ret
1582 1571 SET_SIZE(inb)
1583 1572
1584 1573 #elif defined(__i386)
1585 1574
1586 1575 ENTRY(inb)
1587 1576 subl %eax, %eax
1588 1577 movw PORT(%esp), %dx
1589 1578 inb (%dx)
1590 1579 ret
1591 1580 SET_SIZE(inb)
1592 1581
1593 1582 #endif /* __i386 */
1594 1583 #endif /* __lint */
1595 1584
1596 1585
1597 1586 #if defined(__lint)
1598 1587
1599 1588 /* ARGSUSED */
1600 1589 void
1601 1590 repoutsw(int port, uint16_t *addr, int cnt)
1602 1591 {}
1603 1592
1604 1593 #else /* __lint */
1605 1594
1606 1595 #if defined(__amd64)
1607 1596
1608 1597 ENTRY(repoutsw)
1609 1598 movl %edx, %ecx
1610 1599 movw %di, %dx
1611 1600 rep
1612 1601 D16 outsl
1613 1602 ret
1614 1603 SET_SIZE(repoutsw)
1615 1604
1616 1605 #elif defined(__i386)
1617 1606
1618 1607 /*
1619 1608 * The arguments and saved registers are on the stack in the
1620 1609 * following order:
1621 1610 * | cnt | +16
1622 1611 * | *addr | +12
1623 1612 * | port | +8
1624 1613 * | eip | +4
1625 1614 * | esi | <-- %esp
1626 1615 * If additional values are pushed onto the stack, make sure
1627 1616 * to adjust the following constants accordingly.
1628 1617 */
1629 1618 .set PORT, 8
1630 1619 .set ADDR, 12
1631 1620 .set COUNT, 16
1632 1621
1633 1622 ENTRY(repoutsw)
1634 1623 pushl %esi
1635 1624 movl PORT(%esp), %edx
1636 1625 movl ADDR(%esp), %esi
1637 1626 movl COUNT(%esp), %ecx
1638 1627 rep
1639 1628 D16 outsl
1640 1629 popl %esi
1641 1630 ret
1642 1631 SET_SIZE(repoutsw)
1643 1632
1644 1633 #endif /* __i386 */
1645 1634 #endif /* __lint */
1646 1635
1647 1636
1648 1637 #if defined(__lint)
1649 1638
1650 1639 /* ARGSUSED */
1651 1640 void
1652 1641 repinsw(int port_addr, uint16_t *addr, int cnt)
1653 1642 {}
1654 1643
1655 1644 #else /* __lint */
1656 1645
1657 1646 #if defined(__amd64)
1658 1647
1659 1648 ENTRY(repinsw)
1660 1649 movl %edx, %ecx
1661 1650 movw %di, %dx
1662 1651 rep
1663 1652 D16 insl
1664 1653 ret
1665 1654 SET_SIZE(repinsw)
1666 1655
1667 1656 #elif defined(__i386)
1668 1657
1669 1658 ENTRY(repinsw)
1670 1659 pushl %edi
1671 1660 movl PORT(%esp), %edx
1672 1661 movl ADDR(%esp), %edi
1673 1662 movl COUNT(%esp), %ecx
1674 1663 rep
1675 1664 D16 insl
1676 1665 popl %edi
1677 1666 ret
1678 1667 SET_SIZE(repinsw)
1679 1668
1680 1669 #endif /* __i386 */
1681 1670 #endif /* __lint */
1682 1671
1683 1672
1684 1673 #if defined(__lint)
1685 1674
1686 1675 /* ARGSUSED */
1687 1676 void
1688 1677 repinsb(int port, uint8_t *addr, int count)
1689 1678 {}
1690 1679
1691 1680 #else /* __lint */
1692 1681
1693 1682 #if defined(__amd64)
1694 1683
1695 1684 ENTRY(repinsb)
1696 1685 movl %edx, %ecx
1697 1686 movw %di, %dx
1698 1687 movq %rsi, %rdi
1699 1688 rep
1700 1689 insb
1701 1690 ret
1702 1691 SET_SIZE(repinsb)
1703 1692
1704 1693 #elif defined(__i386)
1705 1694
1706 1695 /*
1707 1696 * The arguments and saved registers are on the stack in the
1708 1697 * following order:
1709 1698 * | cnt | +16
1710 1699 * | *addr | +12
1711 1700 * | port | +8
1712 1701 * | eip | +4
1713 1702 * | esi | <-- %esp
1714 1703 * If additional values are pushed onto the stack, make sure
1715 1704 * to adjust the following constants accordingly.
1716 1705 */
1717 1706 .set IO_PORT, 8
1718 1707 .set IO_ADDR, 12
1719 1708 .set IO_COUNT, 16
1720 1709
1721 1710 ENTRY(repinsb)
1722 1711 pushl %edi
1723 1712 movl IO_ADDR(%esp), %edi
1724 1713 movl IO_COUNT(%esp), %ecx
1725 1714 movl IO_PORT(%esp), %edx
1726 1715 rep
1727 1716 insb
1728 1717 popl %edi
1729 1718 ret
1730 1719 SET_SIZE(repinsb)
1731 1720
1732 1721 #endif /* __i386 */
1733 1722 #endif /* __lint */
1734 1723
1735 1724
1736 1725 /*
1737 1726 * Input a stream of 32-bit words.
1738 1727 * NOTE: count is a DWORD count.
1739 1728 */
1740 1729 #if defined(__lint)
1741 1730
1742 1731 /* ARGSUSED */
1743 1732 void
1744 1733 repinsd(int port, uint32_t *addr, int count)
1745 1734 {}
1746 1735
1747 1736 #else /* __lint */
1748 1737
1749 1738 #if defined(__amd64)
1750 1739
1751 1740 ENTRY(repinsd)
1752 1741 movl %edx, %ecx
1753 1742 movw %di, %dx
1754 1743 movq %rsi, %rdi
1755 1744 rep
1756 1745 insl
1757 1746 ret
1758 1747 SET_SIZE(repinsd)
1759 1748
1760 1749 #elif defined(__i386)
1761 1750
1762 1751 ENTRY(repinsd)
1763 1752 pushl %edi
1764 1753 movl IO_ADDR(%esp), %edi
1765 1754 movl IO_COUNT(%esp), %ecx
1766 1755 movl IO_PORT(%esp), %edx
1767 1756 rep
1768 1757 insl
1769 1758 popl %edi
1770 1759 ret
1771 1760 SET_SIZE(repinsd)
1772 1761
1773 1762 #endif /* __i386 */
1774 1763 #endif /* __lint */
1775 1764
1776 1765 /*
1777 1766 * Output a stream of bytes
1778 1767 * NOTE: count is a byte count
1779 1768 */
1780 1769 #if defined(__lint)
1781 1770
1782 1771 /* ARGSUSED */
1783 1772 void
1784 1773 repoutsb(int port, uint8_t *addr, int count)
1785 1774 {}
1786 1775
1787 1776 #else /* __lint */
1788 1777
1789 1778 #if defined(__amd64)
1790 1779
1791 1780 ENTRY(repoutsb)
1792 1781 movl %edx, %ecx
1793 1782 movw %di, %dx
1794 1783 rep
1795 1784 outsb
1796 1785 ret
1797 1786 SET_SIZE(repoutsb)
1798 1787
1799 1788 #elif defined(__i386)
1800 1789
1801 1790 ENTRY(repoutsb)
1802 1791 pushl %esi
1803 1792 movl IO_ADDR(%esp), %esi
1804 1793 movl IO_COUNT(%esp), %ecx
1805 1794 movl IO_PORT(%esp), %edx
1806 1795 rep
1807 1796 outsb
1808 1797 popl %esi
1809 1798 ret
1810 1799 SET_SIZE(repoutsb)
1811 1800
1812 1801 #endif /* __i386 */
1813 1802 #endif /* __lint */
1814 1803
1815 1804 /*
1816 1805 * Output a stream of 32-bit words
1817 1806 * NOTE: count is a DWORD count
1818 1807 */
1819 1808 #if defined(__lint)
1820 1809
1821 1810 /* ARGSUSED */
1822 1811 void
1823 1812 repoutsd(int port, uint32_t *addr, int count)
1824 1813 {}
1825 1814
1826 1815 #else /* __lint */
1827 1816
1828 1817 #if defined(__amd64)
1829 1818
1830 1819 ENTRY(repoutsd)
1831 1820 movl %edx, %ecx
1832 1821 movw %di, %dx
1833 1822 rep
1834 1823 outsl
1835 1824 ret
1836 1825 SET_SIZE(repoutsd)
1837 1826
1838 1827 #elif defined(__i386)
1839 1828
1840 1829 ENTRY(repoutsd)
1841 1830 pushl %esi
1842 1831 movl IO_ADDR(%esp), %esi
1843 1832 movl IO_COUNT(%esp), %ecx
1844 1833 movl IO_PORT(%esp), %edx
1845 1834 rep
1846 1835 outsl
1847 1836 popl %esi
1848 1837 ret
1849 1838 SET_SIZE(repoutsd)
1850 1839
1851 1840 #endif /* __i386 */
1852 1841 #endif /* __lint */
1853 1842
1854 1843 /*
1855 1844 * void int3(void)
1856 1845 * void int18(void)
1857 1846 * void int20(void)
1858 1847 * void int_cmci(void)
1859 1848 */
1860 1849
1861 1850 #if defined(__lint)
1862 1851
1863 1852 void
1864 1853 int3(void)
1865 1854 {}
1866 1855
1867 1856 void
1868 1857 int18(void)
1869 1858 {}
1870 1859
1871 1860 void
1872 1861 int20(void)
1873 1862 {}
1874 1863
1875 1864 void
1876 1865 int_cmci(void)
1877 1866 {}
1878 1867
1879 1868 #else /* __lint */
1880 1869
1881 1870 ENTRY(int3)
1882 1871 int $T_BPTFLT
1883 1872 ret
1884 1873 SET_SIZE(int3)
1885 1874
1886 1875 ENTRY(int18)
1887 1876 int $T_MCE
1888 1877 ret
1889 1878 SET_SIZE(int18)
1890 1879
1891 1880 ENTRY(int20)
1892 1881 movl boothowto, %eax
1893 1882 andl $RB_DEBUG, %eax
1894 1883 jz 1f
1895 1884
1896 1885 int $T_DBGENTR
1897 1886 1:
1898 1887 rep; ret /* use 2 byte return instruction when branch target */
1899 1888 /* AMD Software Optimization Guide - Section 6.2 */
1900 1889 SET_SIZE(int20)
1901 1890
1902 1891 ENTRY(int_cmci)
1903 1892 int $T_ENOEXTFLT
1904 1893 ret
1905 1894 SET_SIZE(int_cmci)
1906 1895
1907 1896 #endif /* __lint */
1908 1897
1909 1898 #if defined(__lint)
1910 1899
1911 1900 /* ARGSUSED */
1912 1901 int
1913 1902 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1914 1903 { return (0); }
1915 1904
1916 1905 #else /* __lint */
1917 1906
1918 1907 #if defined(__amd64)
1919 1908
1920 1909 ENTRY(scanc)
1921 1910 /* rdi == size */
1922 1911 /* rsi == cp */
1923 1912 /* rdx == table */
1924 1913 /* rcx == mask */
1925 1914 addq %rsi, %rdi /* end = &cp[size] */
1926 1915 .scanloop:
1927 1916 cmpq %rdi, %rsi /* while (cp < end */
1928 1917 jnb .scandone
1929 1918 movzbq (%rsi), %r8 /* %r8 = *cp */
1930 1919 incq %rsi /* cp++ */
1931 1920 testb %cl, (%r8, %rdx)
1932 1921 jz .scanloop /* && (table[*cp] & mask) == 0) */
1933 1922 decq %rsi /* (fix post-increment) */
1934 1923 .scandone:
1935 1924 movl %edi, %eax
1936 1925 subl %esi, %eax /* return (end - cp) */
1937 1926 ret
1938 1927 SET_SIZE(scanc)
1939 1928
1940 1929 #elif defined(__i386)
1941 1930
1942 1931 ENTRY(scanc)
1943 1932 pushl %edi
1944 1933 pushl %esi
1945 1934 movb 24(%esp), %cl /* mask = %cl */
1946 1935 movl 16(%esp), %esi /* cp = %esi */
1947 1936 movl 20(%esp), %edx /* table = %edx */
1948 1937 movl %esi, %edi
1949 1938 addl 12(%esp), %edi /* end = &cp[size]; */
1950 1939 .scanloop:
1951 1940 cmpl %edi, %esi /* while (cp < end */
1952 1941 jnb .scandone
1953 1942 movzbl (%esi), %eax /* %al = *cp */
1954 1943 incl %esi /* cp++ */
1955 1944 movb (%edx, %eax), %al /* %al = table[*cp] */
1956 1945 testb %al, %cl
1957 1946 jz .scanloop /* && (table[*cp] & mask) == 0) */
1958 1947 dec %esi /* post-incremented */
1959 1948 .scandone:
1960 1949 movl %edi, %eax
1961 1950 subl %esi, %eax /* return (end - cp) */
1962 1951 popl %esi
1963 1952 popl %edi
1964 1953 ret
1965 1954 SET_SIZE(scanc)
1966 1955
1967 1956 #endif /* __i386 */
1968 1957 #endif /* __lint */
1969 1958
1970 1959 /*
1971 1960 * Replacement functions for ones that are normally inlined.
1972 1961 * In addition to the copy in i86.il, they are defined here just in case.
1973 1962 */
1974 1963
1975 1964 #if defined(__lint)
1976 1965
1977 1966 ulong_t
1978 1967 intr_clear(void)
1979 1968 { return (0); }
1980 1969
1981 1970 ulong_t
1982 1971 clear_int_flag(void)
1983 1972 { return (0); }
1984 1973
1985 1974 #else /* __lint */
1986 1975
1987 1976 #if defined(__amd64)
1988 1977
1989 1978 ENTRY(intr_clear)
1990 1979 ENTRY(clear_int_flag)
1991 1980 pushfq
1992 1981 popq %rax
1993 1982 #if defined(__xpv)
1994 1983 leaq xpv_panicking, %rdi
1995 1984 movl (%rdi), %edi
1996 1985 cmpl $0, %edi
1997 1986 jne 2f
1998 1987 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1999 1988 /*
2000 1989 * Synthesize the PS_IE bit from the event mask bit
2001 1990 */
2002 1991 andq $_BITNOT(PS_IE), %rax
2003 1992 testb $1, %dl
2004 1993 jnz 1f
2005 1994 orq $PS_IE, %rax
2006 1995 1:
2007 1996 ret
2008 1997 2:
2009 1998 #endif
2010 1999 CLI(%rdi)
2011 2000 ret
2012 2001 SET_SIZE(clear_int_flag)
2013 2002 SET_SIZE(intr_clear)
2014 2003
2015 2004 #elif defined(__i386)
2016 2005
2017 2006 ENTRY(intr_clear)
2018 2007 ENTRY(clear_int_flag)
2019 2008 pushfl
2020 2009 popl %eax
2021 2010 #if defined(__xpv)
2022 2011 leal xpv_panicking, %edx
2023 2012 movl (%edx), %edx
2024 2013 cmpl $0, %edx
2025 2014 jne 2f
2026 2015 CLIRET(%edx, %cl) /* returns event mask in %cl */
2027 2016 /*
2028 2017 * Synthesize the PS_IE bit from the event mask bit
2029 2018 */
2030 2019 andl $_BITNOT(PS_IE), %eax
2031 2020 testb $1, %cl
2032 2021 jnz 1f
2033 2022 orl $PS_IE, %eax
2034 2023 1:
2035 2024 ret
2036 2025 2:
2037 2026 #endif
2038 2027 CLI(%edx)
2039 2028 ret
2040 2029 SET_SIZE(clear_int_flag)
2041 2030 SET_SIZE(intr_clear)
2042 2031
2043 2032 #endif /* __i386 */
2044 2033 #endif /* __lint */
2045 2034
2046 2035 #if defined(__lint)
2047 2036
2048 2037 struct cpu *
2049 2038 curcpup(void)
2050 2039 { return 0; }
2051 2040
2052 2041 #else /* __lint */
2053 2042
2054 2043 #if defined(__amd64)
2055 2044
2056 2045 ENTRY(curcpup)
2057 2046 movq %gs:CPU_SELF, %rax
2058 2047 ret
2059 2048 SET_SIZE(curcpup)
2060 2049
2061 2050 #elif defined(__i386)
2062 2051
2063 2052 ENTRY(curcpup)
2064 2053 movl %gs:CPU_SELF, %eax
2065 2054 ret
2066 2055 SET_SIZE(curcpup)
2067 2056
2068 2057 #endif /* __i386 */
2069 2058 #endif /* __lint */
2070 2059
2071 2060 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2072 2061 * These functions reverse the byte order of the input parameter and returns
2073 2062 * the result. This is to convert the byte order from host byte order
2074 2063 * (little endian) to network byte order (big endian), or vice versa.
2075 2064 */
2076 2065
2077 2066 #if defined(__lint)
2078 2067
2079 2068 uint64_t
2080 2069 htonll(uint64_t i)
2081 2070 { return (i); }
2082 2071
2083 2072 uint64_t
2084 2073 ntohll(uint64_t i)
2085 2074 { return (i); }
2086 2075
2087 2076 uint32_t
2088 2077 htonl(uint32_t i)
2089 2078 { return (i); }
2090 2079
2091 2080 uint32_t
2092 2081 ntohl(uint32_t i)
2093 2082 { return (i); }
2094 2083
2095 2084 uint16_t
2096 2085 htons(uint16_t i)
2097 2086 { return (i); }
2098 2087
2099 2088 uint16_t
2100 2089 ntohs(uint16_t i)
2101 2090 { return (i); }
2102 2091
2103 2092 #else /* __lint */
2104 2093
2105 2094 #if defined(__amd64)
2106 2095
2107 2096 ENTRY(htonll)
2108 2097 ALTENTRY(ntohll)
2109 2098 movq %rdi, %rax
2110 2099 bswapq %rax
2111 2100 ret
2112 2101 SET_SIZE(ntohll)
2113 2102 SET_SIZE(htonll)
2114 2103
2115 2104 /* XX64 there must be shorter sequences for this */
2116 2105 ENTRY(htonl)
2117 2106 ALTENTRY(ntohl)
2118 2107 movl %edi, %eax
2119 2108 bswap %eax
2120 2109 ret
2121 2110 SET_SIZE(ntohl)
2122 2111 SET_SIZE(htonl)
2123 2112
2124 2113 /* XX64 there must be better sequences for this */
2125 2114 ENTRY(htons)
2126 2115 ALTENTRY(ntohs)
2127 2116 movl %edi, %eax
2128 2117 bswap %eax
2129 2118 shrl $16, %eax
2130 2119 ret
2131 2120 SET_SIZE(ntohs)
2132 2121 SET_SIZE(htons)
2133 2122
2134 2123 #elif defined(__i386)
2135 2124
2136 2125 ENTRY(htonll)
2137 2126 ALTENTRY(ntohll)
2138 2127 movl 4(%esp), %edx
2139 2128 movl 8(%esp), %eax
2140 2129 bswap %edx
2141 2130 bswap %eax
2142 2131 ret
2143 2132 SET_SIZE(ntohll)
2144 2133 SET_SIZE(htonll)
2145 2134
2146 2135 ENTRY(htonl)
2147 2136 ALTENTRY(ntohl)
2148 2137 movl 4(%esp), %eax
2149 2138 bswap %eax
2150 2139 ret
2151 2140 SET_SIZE(ntohl)
2152 2141 SET_SIZE(htonl)
2153 2142
2154 2143 ENTRY(htons)
2155 2144 ALTENTRY(ntohs)
2156 2145 movl 4(%esp), %eax
2157 2146 bswap %eax
2158 2147 shrl $16, %eax
2159 2148 ret
2160 2149 SET_SIZE(ntohs)
2161 2150 SET_SIZE(htons)
2162 2151
2163 2152 #endif /* __i386 */
2164 2153 #endif /* __lint */
2165 2154
2166 2155
2167 2156 #if defined(__lint)
2168 2157
2169 2158 /* ARGSUSED */
2170 2159 void
2171 2160 intr_restore(ulong_t i)
2172 2161 { return; }
2173 2162
2174 2163 /* ARGSUSED */
2175 2164 void
2176 2165 restore_int_flag(ulong_t i)
2177 2166 { return; }
2178 2167
2179 2168 #else /* __lint */
2180 2169
2181 2170 #if defined(__amd64)
2182 2171
2183 2172 ENTRY(intr_restore)
2184 2173 ENTRY(restore_int_flag)
2185 2174 testq $PS_IE, %rdi
2186 2175 jz 1f
2187 2176 #if defined(__xpv)
2188 2177 leaq xpv_panicking, %rsi
2189 2178 movl (%rsi), %esi
2190 2179 cmpl $0, %esi
2191 2180 jne 1f
2192 2181 /*
2193 2182 * Since we're -really- running unprivileged, our attempt
2194 2183 * to change the state of the IF bit will be ignored.
2195 2184 * The virtual IF bit is tweaked by CLI and STI.
2196 2185 */
2197 2186 IE_TO_EVENT_MASK(%rsi, %rdi)
2198 2187 #else
2199 2188 sti
2200 2189 #endif
2201 2190 1:
2202 2191 ret
2203 2192 SET_SIZE(restore_int_flag)
2204 2193 SET_SIZE(intr_restore)
2205 2194
2206 2195 #elif defined(__i386)
2207 2196
2208 2197 ENTRY(intr_restore)
2209 2198 ENTRY(restore_int_flag)
2210 2199 testl $PS_IE, 4(%esp)
2211 2200 jz 1f
2212 2201 #if defined(__xpv)
2213 2202 leal xpv_panicking, %edx
2214 2203 movl (%edx), %edx
2215 2204 cmpl $0, %edx
2216 2205 jne 1f
2217 2206 /*
2218 2207 * Since we're -really- running unprivileged, our attempt
2219 2208 * to change the state of the IF bit will be ignored.
2220 2209 * The virtual IF bit is tweaked by CLI and STI.
2221 2210 */
2222 2211 IE_TO_EVENT_MASK(%edx, 4(%esp))
2223 2212 #else
2224 2213 sti
2225 2214 #endif
2226 2215 1:
2227 2216 ret
2228 2217 SET_SIZE(restore_int_flag)
2229 2218 SET_SIZE(intr_restore)
2230 2219
2231 2220 #endif /* __i386 */
2232 2221 #endif /* __lint */
2233 2222
2234 2223 #if defined(__lint)
2235 2224
2236 2225 void
2237 2226 sti(void)
2238 2227 {}
2239 2228
2240 2229 void
2241 2230 cli(void)
2242 2231 {}
2243 2232
2244 2233 #else /* __lint */
2245 2234
2246 2235 ENTRY(sti)
2247 2236 STI
2248 2237 ret
2249 2238 SET_SIZE(sti)
2250 2239
2251 2240 ENTRY(cli)
2252 2241 #if defined(__amd64)
2253 2242 CLI(%rax)
2254 2243 #elif defined(__i386)
2255 2244 CLI(%eax)
2256 2245 #endif /* __i386 */
2257 2246 ret
2258 2247 SET_SIZE(cli)
2259 2248
2260 2249 #endif /* __lint */
2261 2250
2262 2251 #if defined(__lint)
2263 2252
2264 2253 dtrace_icookie_t
2265 2254 dtrace_interrupt_disable(void)
2266 2255 { return (0); }
2267 2256
2268 2257 #else /* __lint */
2269 2258
2270 2259 #if defined(__amd64)
2271 2260
2272 2261 ENTRY(dtrace_interrupt_disable)
2273 2262 pushfq
2274 2263 popq %rax
2275 2264 #if defined(__xpv)
2276 2265 leaq xpv_panicking, %rdi
2277 2266 movl (%rdi), %edi
2278 2267 cmpl $0, %edi
2279 2268 jne .dtrace_interrupt_disable_done
2280 2269 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2281 2270 /*
2282 2271 * Synthesize the PS_IE bit from the event mask bit
2283 2272 */
2284 2273 andq $_BITNOT(PS_IE), %rax
2285 2274 testb $1, %dl
2286 2275 jnz .dtrace_interrupt_disable_done
2287 2276 orq $PS_IE, %rax
2288 2277 #else
2289 2278 CLI(%rdx)
2290 2279 #endif
2291 2280 .dtrace_interrupt_disable_done:
2292 2281 ret
2293 2282 SET_SIZE(dtrace_interrupt_disable)
2294 2283
2295 2284 #elif defined(__i386)
2296 2285
2297 2286 ENTRY(dtrace_interrupt_disable)
2298 2287 pushfl
2299 2288 popl %eax
2300 2289 #if defined(__xpv)
2301 2290 leal xpv_panicking, %edx
2302 2291 movl (%edx), %edx
2303 2292 cmpl $0, %edx
2304 2293 jne .dtrace_interrupt_disable_done
2305 2294 CLIRET(%edx, %cl) /* returns event mask in %cl */
2306 2295 /*
2307 2296 * Synthesize the PS_IE bit from the event mask bit
2308 2297 */
2309 2298 andl $_BITNOT(PS_IE), %eax
2310 2299 testb $1, %cl
2311 2300 jnz .dtrace_interrupt_disable_done
2312 2301 orl $PS_IE, %eax
2313 2302 #else
2314 2303 CLI(%edx)
2315 2304 #endif
2316 2305 .dtrace_interrupt_disable_done:
2317 2306 ret
2318 2307 SET_SIZE(dtrace_interrupt_disable)
2319 2308
2320 2309 #endif /* __i386 */
2321 2310 #endif /* __lint */
2322 2311
2323 2312 #if defined(__lint)
2324 2313
2325 2314 /*ARGSUSED*/
2326 2315 void
2327 2316 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2328 2317 {}
2329 2318
2330 2319 #else /* __lint */
2331 2320
2332 2321 #if defined(__amd64)
2333 2322
2334 2323 ENTRY(dtrace_interrupt_enable)
2335 2324 pushq %rdi
2336 2325 popfq
2337 2326 #if defined(__xpv)
2338 2327 leaq xpv_panicking, %rdx
2339 2328 movl (%rdx), %edx
2340 2329 cmpl $0, %edx
2341 2330 jne .dtrace_interrupt_enable_done
2342 2331 /*
2343 2332 * Since we're -really- running unprivileged, our attempt
2344 2333 * to change the state of the IF bit will be ignored. The
2345 2334 * virtual IF bit is tweaked by CLI and STI.
2346 2335 */
2347 2336 IE_TO_EVENT_MASK(%rdx, %rdi)
2348 2337 #endif
2349 2338 .dtrace_interrupt_enable_done:
2350 2339 ret
2351 2340 SET_SIZE(dtrace_interrupt_enable)
2352 2341
2353 2342 #elif defined(__i386)
2354 2343
2355 2344 ENTRY(dtrace_interrupt_enable)
2356 2345 movl 4(%esp), %eax
2357 2346 pushl %eax
2358 2347 popfl
2359 2348 #if defined(__xpv)
2360 2349 leal xpv_panicking, %edx
2361 2350 movl (%edx), %edx
2362 2351 cmpl $0, %edx
2363 2352 jne .dtrace_interrupt_enable_done
2364 2353 /*
2365 2354 * Since we're -really- running unprivileged, our attempt
2366 2355 * to change the state of the IF bit will be ignored. The
2367 2356 * virtual IF bit is tweaked by CLI and STI.
2368 2357 */
2369 2358 IE_TO_EVENT_MASK(%edx, %eax)
2370 2359 #endif
2371 2360 .dtrace_interrupt_enable_done:
2372 2361 ret
2373 2362 SET_SIZE(dtrace_interrupt_enable)
2374 2363
2375 2364 #endif /* __i386 */
2376 2365 #endif /* __lint */
2377 2366
2378 2367
2379 2368 #if defined(lint)
2380 2369
2381 2370 void
2382 2371 dtrace_membar_producer(void)
2383 2372 {}
2384 2373
2385 2374 void
2386 2375 dtrace_membar_consumer(void)
2387 2376 {}
2388 2377
2389 2378 #else /* __lint */
2390 2379
2391 2380 ENTRY(dtrace_membar_producer)
2392 2381 rep; ret /* use 2 byte return instruction when branch target */
2393 2382 /* AMD Software Optimization Guide - Section 6.2 */
2394 2383 SET_SIZE(dtrace_membar_producer)
2395 2384
2396 2385 ENTRY(dtrace_membar_consumer)
2397 2386 rep; ret /* use 2 byte return instruction when branch target */
2398 2387 /* AMD Software Optimization Guide - Section 6.2 */
2399 2388 SET_SIZE(dtrace_membar_consumer)
2400 2389
2401 2390 #endif /* __lint */
2402 2391
2403 2392 #if defined(__lint)
2404 2393
2405 2394 kthread_id_t
2406 2395 threadp(void)
2407 2396 { return ((kthread_id_t)0); }
2408 2397
2409 2398 #else /* __lint */
2410 2399
2411 2400 #if defined(__amd64)
2412 2401
2413 2402 ENTRY(threadp)
2414 2403 movq %gs:CPU_THREAD, %rax
2415 2404 ret
2416 2405 SET_SIZE(threadp)
2417 2406
2418 2407 #elif defined(__i386)
2419 2408
2420 2409 ENTRY(threadp)
2421 2410 movl %gs:CPU_THREAD, %eax
2422 2411 ret
2423 2412 SET_SIZE(threadp)
2424 2413
2425 2414 #endif /* __i386 */
2426 2415 #endif /* __lint */
2427 2416
2428 2417 /*
2429 2418 * Checksum routine for Internet Protocol Headers
2430 2419 */
2431 2420
2432 2421 #if defined(__lint)
2433 2422
2434 2423 /* ARGSUSED */
2435 2424 unsigned int
2436 2425 ip_ocsum(
2437 2426 ushort_t *address, /* ptr to 1st message buffer */
2438 2427 int halfword_count, /* length of data */
2439 2428 unsigned int sum) /* partial checksum */
2440 2429 {
2441 2430 int i;
2442 2431 unsigned int psum = 0; /* partial sum */
2443 2432
2444 2433 for (i = 0; i < halfword_count; i++, address++) {
2445 2434 psum += *address;
2446 2435 }
2447 2436
2448 2437 while ((psum >> 16) != 0) {
2449 2438 psum = (psum & 0xffff) + (psum >> 16);
2450 2439 }
2451 2440
2452 2441 psum += sum;
2453 2442
2454 2443 while ((psum >> 16) != 0) {
2455 2444 psum = (psum & 0xffff) + (psum >> 16);
2456 2445 }
2457 2446
2458 2447 return (psum);
2459 2448 }
2460 2449
2461 2450 #else /* __lint */
2462 2451
2463 2452 #if defined(__amd64)
2464 2453
2465 2454 ENTRY(ip_ocsum)
2466 2455 pushq %rbp
2467 2456 movq %rsp, %rbp
2468 2457 #ifdef DEBUG
2469 2458 movq postbootkernelbase(%rip), %rax
2470 2459 cmpq %rax, %rdi
2471 2460 jnb 1f
2472 2461 xorl %eax, %eax
2473 2462 movq %rdi, %rsi
2474 2463 leaq .ip_ocsum_panic_msg(%rip), %rdi
2475 2464 call panic
2476 2465 /*NOTREACHED*/
2477 2466 .ip_ocsum_panic_msg:
2478 2467 .string "ip_ocsum: address 0x%p below kernelbase\n"
2479 2468 1:
2480 2469 #endif
2481 2470 movl %esi, %ecx /* halfword_count */
2482 2471 movq %rdi, %rsi /* address */
2483 2472 /* partial sum in %edx */
2484 2473 xorl %eax, %eax
2485 2474 testl %ecx, %ecx
2486 2475 jz .ip_ocsum_done
2487 2476 testq $3, %rsi
2488 2477 jnz .ip_csum_notaligned
2489 2478 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2490 2479 .next_iter:
2491 2480 /* XX64 opportunities for prefetch? */
2492 2481 /* XX64 compute csum with 64 bit quantities? */
2493 2482 subl $32, %ecx
2494 2483 jl .less_than_32
2495 2484
2496 2485 addl 0(%rsi), %edx
2497 2486 .only60:
2498 2487 adcl 4(%rsi), %eax
2499 2488 .only56:
2500 2489 adcl 8(%rsi), %edx
2501 2490 .only52:
2502 2491 adcl 12(%rsi), %eax
2503 2492 .only48:
2504 2493 adcl 16(%rsi), %edx
2505 2494 .only44:
2506 2495 adcl 20(%rsi), %eax
2507 2496 .only40:
2508 2497 adcl 24(%rsi), %edx
2509 2498 .only36:
2510 2499 adcl 28(%rsi), %eax
2511 2500 .only32:
2512 2501 adcl 32(%rsi), %edx
2513 2502 .only28:
2514 2503 adcl 36(%rsi), %eax
2515 2504 .only24:
2516 2505 adcl 40(%rsi), %edx
2517 2506 .only20:
2518 2507 adcl 44(%rsi), %eax
2519 2508 .only16:
2520 2509 adcl 48(%rsi), %edx
2521 2510 .only12:
2522 2511 adcl 52(%rsi), %eax
2523 2512 .only8:
2524 2513 adcl 56(%rsi), %edx
2525 2514 .only4:
2526 2515 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2527 2516 .only0:
2528 2517 adcl $0, %eax /* could be adding -1 in eax with a carry */
2529 2518 adcl $0, %eax
2530 2519
2531 2520 addq $64, %rsi
2532 2521 testl %ecx, %ecx
2533 2522 jnz .next_iter
2534 2523
2535 2524 .ip_ocsum_done:
2536 2525 addl %eax, %edx
2537 2526 adcl $0, %edx
2538 2527 movl %edx, %eax /* form a 16 bit checksum by */
2539 2528 shrl $16, %eax /* adding two halves of 32 bit checksum */
2540 2529 addw %dx, %ax
2541 2530 adcw $0, %ax
2542 2531 andl $0xffff, %eax
2543 2532 leave
2544 2533 ret
2545 2534
2546 2535 .ip_csum_notaligned:
2547 2536 xorl %edi, %edi
2548 2537 movw (%rsi), %di
2549 2538 addl %edi, %edx
2550 2539 adcl $0, %edx
2551 2540 addq $2, %rsi
2552 2541 decl %ecx
2553 2542 jmp .ip_csum_aligned
2554 2543
2555 2544 .less_than_32:
2556 2545 addl $32, %ecx
2557 2546 testl $1, %ecx
2558 2547 jz .size_aligned
2559 2548 andl $0xfe, %ecx
2560 2549 movzwl (%rsi, %rcx, 2), %edi
2561 2550 addl %edi, %edx
2562 2551 adcl $0, %edx
2563 2552 .size_aligned:
2564 2553 movl %ecx, %edi
2565 2554 shrl $1, %ecx
2566 2555 shl $1, %edi
2567 2556 subq $64, %rdi
2568 2557 addq %rdi, %rsi
2569 2558 leaq .ip_ocsum_jmptbl(%rip), %rdi
2570 2559 leaq (%rdi, %rcx, 8), %rdi
2571 2560 xorl %ecx, %ecx
2572 2561 clc
2573 2562 jmp *(%rdi)
2574 2563
2575 2564 .align 8
2576 2565 .ip_ocsum_jmptbl:
2577 2566 .quad .only0, .only4, .only8, .only12, .only16, .only20
2578 2567 .quad .only24, .only28, .only32, .only36, .only40, .only44
2579 2568 .quad .only48, .only52, .only56, .only60
2580 2569 SET_SIZE(ip_ocsum)
2581 2570
2582 2571 #elif defined(__i386)
2583 2572
2584 2573 ENTRY(ip_ocsum)
2585 2574 pushl %ebp
2586 2575 movl %esp, %ebp
2587 2576 pushl %ebx
2588 2577 pushl %esi
2589 2578 pushl %edi
2590 2579 movl 12(%ebp), %ecx /* count of half words */
2591 2580 movl 16(%ebp), %edx /* partial checksum */
2592 2581 movl 8(%ebp), %esi
2593 2582 xorl %eax, %eax
2594 2583 testl %ecx, %ecx
2595 2584 jz .ip_ocsum_done
2596 2585
2597 2586 testl $3, %esi
2598 2587 jnz .ip_csum_notaligned
2599 2588 .ip_csum_aligned:
2600 2589 .next_iter:
2601 2590 subl $32, %ecx
2602 2591 jl .less_than_32
2603 2592
2604 2593 addl 0(%esi), %edx
2605 2594 .only60:
2606 2595 adcl 4(%esi), %eax
2607 2596 .only56:
2608 2597 adcl 8(%esi), %edx
2609 2598 .only52:
2610 2599 adcl 12(%esi), %eax
2611 2600 .only48:
2612 2601 adcl 16(%esi), %edx
2613 2602 .only44:
2614 2603 adcl 20(%esi), %eax
2615 2604 .only40:
2616 2605 adcl 24(%esi), %edx
2617 2606 .only36:
2618 2607 adcl 28(%esi), %eax
2619 2608 .only32:
2620 2609 adcl 32(%esi), %edx
2621 2610 .only28:
2622 2611 adcl 36(%esi), %eax
2623 2612 .only24:
2624 2613 adcl 40(%esi), %edx
2625 2614 .only20:
2626 2615 adcl 44(%esi), %eax
2627 2616 .only16:
2628 2617 adcl 48(%esi), %edx
2629 2618 .only12:
2630 2619 adcl 52(%esi), %eax
2631 2620 .only8:
2632 2621 adcl 56(%esi), %edx
2633 2622 .only4:
2634 2623 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2635 2624 .only0:
2636 2625 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2637 2626 adcl $0, %eax
2638 2627
2639 2628 addl $64, %esi
2640 2629 andl %ecx, %ecx
2641 2630 jnz .next_iter
2642 2631
2643 2632 .ip_ocsum_done:
2644 2633 addl %eax, %edx
2645 2634 adcl $0, %edx
2646 2635 movl %edx, %eax /* form a 16 bit checksum by */
2647 2636 shrl $16, %eax /* adding two halves of 32 bit checksum */
2648 2637 addw %dx, %ax
2649 2638 adcw $0, %ax
2650 2639 andl $0xffff, %eax
2651 2640 popl %edi /* restore registers */
2652 2641 popl %esi
2653 2642 popl %ebx
2654 2643 leave
2655 2644 ret
2656 2645
2657 2646 .ip_csum_notaligned:
2658 2647 xorl %edi, %edi
2659 2648 movw (%esi), %di
2660 2649 addl %edi, %edx
2661 2650 adcl $0, %edx
2662 2651 addl $2, %esi
2663 2652 decl %ecx
2664 2653 jmp .ip_csum_aligned
2665 2654
2666 2655 .less_than_32:
2667 2656 addl $32, %ecx
2668 2657 testl $1, %ecx
2669 2658 jz .size_aligned
2670 2659 andl $0xfe, %ecx
2671 2660 movzwl (%esi, %ecx, 2), %edi
2672 2661 addl %edi, %edx
2673 2662 adcl $0, %edx
2674 2663 .size_aligned:
2675 2664 movl %ecx, %edi
2676 2665 shrl $1, %ecx
2677 2666 shl $1, %edi
2678 2667 subl $64, %edi
2679 2668 addl %edi, %esi
2680 2669 movl $.ip_ocsum_jmptbl, %edi
2681 2670 lea (%edi, %ecx, 4), %edi
2682 2671 xorl %ecx, %ecx
2683 2672 clc
2684 2673 jmp *(%edi)
2685 2674 SET_SIZE(ip_ocsum)
2686 2675
2687 2676 .data
2688 2677 .align 4
2689 2678
2690 2679 .ip_ocsum_jmptbl:
2691 2680 .long .only0, .only4, .only8, .only12, .only16, .only20
2692 2681 .long .only24, .only28, .only32, .only36, .only40, .only44
2693 2682 .long .only48, .only52, .only56, .only60
2694 2683
2695 2684
2696 2685 #endif /* __i386 */
2697 2686 #endif /* __lint */
2698 2687
2699 2688 /*
2700 2689 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2701 2690 * Provided to manipulate hrtime_t values.
2702 2691 */
2703 2692 #if defined(__lint)
2704 2693
2705 2694 /* result = a * b; */
2706 2695
2707 2696 /* ARGSUSED */
2708 2697 unsigned long long
2709 2698 mul32(uint_t a, uint_t b)
2710 2699 { return (0); }
2711 2700
2712 2701 #else /* __lint */
2713 2702
2714 2703 #if defined(__amd64)
2715 2704
2716 2705 ENTRY(mul32)
2717 2706 xorl %edx, %edx /* XX64 joe, paranoia? */
2718 2707 movl %edi, %eax
2719 2708 mull %esi
2720 2709 shlq $32, %rdx
2721 2710 orq %rdx, %rax
2722 2711 ret
2723 2712 SET_SIZE(mul32)
2724 2713
2725 2714 #elif defined(__i386)
2726 2715
2727 2716 ENTRY(mul32)
2728 2717 movl 8(%esp), %eax
2729 2718 movl 4(%esp), %ecx
2730 2719 mull %ecx
2731 2720 ret
2732 2721 SET_SIZE(mul32)
2733 2722
2734 2723 #endif /* __i386 */
2735 2724 #endif /* __lint */
2736 2725
2737 2726 #if defined(notused)
2738 2727 #if defined(__lint)
2739 2728 /* ARGSUSED */
2740 2729 void
2741 2730 load_pte64(uint64_t *pte, uint64_t pte_value)
2742 2731 {}
2743 2732 #else /* __lint */
2744 2733 .globl load_pte64
2745 2734 load_pte64:
2746 2735 movl 4(%esp), %eax
2747 2736 movl 8(%esp), %ecx
2748 2737 movl 12(%esp), %edx
2749 2738 movl %edx, 4(%eax)
2750 2739 movl %ecx, (%eax)
2751 2740 ret
2752 2741 #endif /* __lint */
2753 2742 #endif /* notused */
2754 2743
2755 2744 #if defined(__lint)
2756 2745
2757 2746 /*ARGSUSED*/
2758 2747 void
2759 2748 scan_memory(caddr_t addr, size_t size)
2760 2749 {}
2761 2750
2762 2751 #else /* __lint */
2763 2752
2764 2753 #if defined(__amd64)
2765 2754
2766 2755 ENTRY(scan_memory)
2767 2756 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2768 2757 jz .scanm_done
2769 2758 movq %rsi, %rcx /* move count into rep control register */
2770 2759 movq %rdi, %rsi /* move addr into lodsq control reg. */
2771 2760 rep lodsq /* scan the memory range */
2772 2761 .scanm_done:
2773 2762 rep; ret /* use 2 byte return instruction when branch target */
2774 2763 /* AMD Software Optimization Guide - Section 6.2 */
2775 2764 SET_SIZE(scan_memory)
2776 2765
2777 2766 #elif defined(__i386)
2778 2767
2779 2768 ENTRY(scan_memory)
2780 2769 pushl %ecx
2781 2770 pushl %esi
2782 2771 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2783 2772 shrl $2, %ecx /* convert from byte count to word count */
2784 2773 jz .scanm_done
2785 2774 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2786 2775 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2787 2776 lodsl
2788 2777 .scanm_done:
2789 2778 popl %esi
2790 2779 popl %ecx
2791 2780 ret
2792 2781 SET_SIZE(scan_memory)
2793 2782
2794 2783 #endif /* __i386 */
2795 2784 #endif /* __lint */
2796 2785
2797 2786
2798 2787 #if defined(__lint)
2799 2788
2800 2789 /*ARGSUSED */
2801 2790 int
2802 2791 lowbit(ulong_t i)
2803 2792 { return (0); }
2804 2793
2805 2794 #else /* __lint */
2806 2795
2807 2796 #if defined(__amd64)
2808 2797
2809 2798 ENTRY(lowbit)
2810 2799 movl $-1, %eax
2811 2800 bsfq %rdi, %rdi
2812 2801 cmovnz %edi, %eax
2813 2802 incl %eax
2814 2803 ret
2815 2804 SET_SIZE(lowbit)
2816 2805
2817 2806 #elif defined(__i386)
2818 2807
2819 2808 ENTRY(lowbit)
2820 2809 bsfl 4(%esp), %eax
2821 2810 jz 0f
2822 2811 incl %eax
2823 2812 ret
2824 2813 0:
2825 2814 xorl %eax, %eax
2826 2815 ret
2827 2816 SET_SIZE(lowbit)
2828 2817
2829 2818 #endif /* __i386 */
2830 2819 #endif /* __lint */
2831 2820
2832 2821 #if defined(__lint)
2833 2822
2834 2823 /*ARGSUSED*/
2835 2824 int
2836 2825 highbit(ulong_t i)
2837 2826 { return (0); }
2838 2827
2839 2828 /*ARGSUSED*/
2840 2829 int
2841 2830 highbit64(uint64_t i)
2842 2831 { return (0); }
2843 2832
2844 2833 #else /* __lint */
2845 2834
2846 2835 #if defined(__amd64)
2847 2836
2848 2837 ENTRY(highbit)
2849 2838 ALTENTRY(highbit64)
2850 2839 movl $-1, %eax
2851 2840 bsrq %rdi, %rdi
2852 2841 cmovnz %edi, %eax
2853 2842 incl %eax
2854 2843 ret
2855 2844 SET_SIZE(highbit64)
2856 2845 SET_SIZE(highbit)
2857 2846
2858 2847 #elif defined(__i386)
2859 2848
2860 2849 ENTRY(highbit)
2861 2850 bsrl 4(%esp), %eax
2862 2851 jz 0f
2863 2852 incl %eax
2864 2853 ret
2865 2854 0:
2866 2855 xorl %eax, %eax
2867 2856 ret
2868 2857 SET_SIZE(highbit)
2869 2858
2870 2859 ENTRY(highbit64)
2871 2860 bsrl 8(%esp), %eax
2872 2861 jz highbit
2873 2862 addl $33, %eax
2874 2863 ret
2875 2864 SET_SIZE(highbit64)
2876 2865
2877 2866 #endif /* __i386 */
2878 2867 #endif /* __lint */
2879 2868
2880 2869 #if defined(__lint)
2881 2870
2882 2871 /*ARGSUSED*/
2883 2872 uint64_t
2884 2873 rdmsr(uint_t r)
2885 2874 { return (0); }
2886 2875
2887 2876 /*ARGSUSED*/
2888 2877 void
2889 2878 wrmsr(uint_t r, const uint64_t val)
2890 2879 {}
2891 2880
2892 2881 /*ARGSUSED*/
2893 2882 uint64_t
2894 2883 xrdmsr(uint_t r)
2895 2884 { return (0); }
2896 2885
2897 2886 /*ARGSUSED*/
2898 2887 void
2899 2888 xwrmsr(uint_t r, const uint64_t val)
2900 2889 {}
2901 2890
2902 2891 void
2903 2892 invalidate_cache(void)
2904 2893 {}
2905 2894
2906 2895 /*ARGSUSED*/
2907 2896 uint64_t
2908 2897 get_xcr(uint_t r)
2909 2898 { return (0); }
2910 2899
2911 2900 /*ARGSUSED*/
2912 2901 void
2913 2902 set_xcr(uint_t r, const uint64_t val)
2914 2903 {}
2915 2904
2916 2905 #else /* __lint */
2917 2906
2918 2907 #define XMSR_ACCESS_VAL $0x9c5a203a
2919 2908
2920 2909 #if defined(__amd64)
2921 2910
2922 2911 ENTRY(rdmsr)
2923 2912 movl %edi, %ecx
2924 2913 rdmsr
2925 2914 shlq $32, %rdx
2926 2915 orq %rdx, %rax
2927 2916 ret
2928 2917 SET_SIZE(rdmsr)
2929 2918
2930 2919 ENTRY(wrmsr)
2931 2920 movq %rsi, %rdx
2932 2921 shrq $32, %rdx
2933 2922 movl %esi, %eax
2934 2923 movl %edi, %ecx
2935 2924 wrmsr
2936 2925 ret
2937 2926 SET_SIZE(wrmsr)
2938 2927
2939 2928 ENTRY(xrdmsr)
2940 2929 pushq %rbp
2941 2930 movq %rsp, %rbp
2942 2931 movl %edi, %ecx
2943 2932 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2944 2933 rdmsr
2945 2934 shlq $32, %rdx
2946 2935 orq %rdx, %rax
2947 2936 leave
2948 2937 ret
2949 2938 SET_SIZE(xrdmsr)
2950 2939
2951 2940 ENTRY(xwrmsr)
2952 2941 pushq %rbp
2953 2942 movq %rsp, %rbp
2954 2943 movl %edi, %ecx
2955 2944 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2956 2945 movq %rsi, %rdx
2957 2946 shrq $32, %rdx
2958 2947 movl %esi, %eax
2959 2948 wrmsr
2960 2949 leave
2961 2950 ret
2962 2951 SET_SIZE(xwrmsr)
2963 2952
2964 2953 ENTRY(get_xcr)
2965 2954 movl %edi, %ecx
2966 2955 #xgetbv
2967 2956 .byte 0x0f,0x01,0xd0
2968 2957 shlq $32, %rdx
2969 2958 orq %rdx, %rax
2970 2959 ret
2971 2960 SET_SIZE(get_xcr)
2972 2961
2973 2962 ENTRY(set_xcr)
2974 2963 movq %rsi, %rdx
2975 2964 shrq $32, %rdx
2976 2965 movl %esi, %eax
2977 2966 movl %edi, %ecx
2978 2967 #xsetbv
2979 2968 .byte 0x0f,0x01,0xd1
2980 2969 ret
2981 2970 SET_SIZE(set_xcr)
2982 2971
2983 2972 #elif defined(__i386)
2984 2973
2985 2974 ENTRY(rdmsr)
2986 2975 movl 4(%esp), %ecx
2987 2976 rdmsr
2988 2977 ret
2989 2978 SET_SIZE(rdmsr)
2990 2979
2991 2980 ENTRY(wrmsr)
2992 2981 movl 4(%esp), %ecx
2993 2982 movl 8(%esp), %eax
2994 2983 movl 12(%esp), %edx
2995 2984 wrmsr
2996 2985 ret
2997 2986 SET_SIZE(wrmsr)
2998 2987
2999 2988 ENTRY(xrdmsr)
3000 2989 pushl %ebp
3001 2990 movl %esp, %ebp
3002 2991 movl 8(%esp), %ecx
3003 2992 pushl %edi
3004 2993 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3005 2994 rdmsr
3006 2995 popl %edi
3007 2996 leave
3008 2997 ret
3009 2998 SET_SIZE(xrdmsr)
3010 2999
3011 3000 ENTRY(xwrmsr)
3012 3001 pushl %ebp
3013 3002 movl %esp, %ebp
3014 3003 movl 8(%esp), %ecx
3015 3004 movl 12(%esp), %eax
3016 3005 movl 16(%esp), %edx
3017 3006 pushl %edi
3018 3007 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3019 3008 wrmsr
3020 3009 popl %edi
3021 3010 leave
3022 3011 ret
3023 3012 SET_SIZE(xwrmsr)
3024 3013
3025 3014 ENTRY(get_xcr)
3026 3015 movl 4(%esp), %ecx
3027 3016 #xgetbv
3028 3017 .byte 0x0f,0x01,0xd0
3029 3018 ret
3030 3019 SET_SIZE(get_xcr)
3031 3020
3032 3021 ENTRY(set_xcr)
3033 3022 movl 4(%esp), %ecx
3034 3023 movl 8(%esp), %eax
3035 3024 movl 12(%esp), %edx
3036 3025 #xsetbv
3037 3026 .byte 0x0f,0x01,0xd1
3038 3027 ret
3039 3028 SET_SIZE(set_xcr)
3040 3029
3041 3030 #endif /* __i386 */
3042 3031
3043 3032 ENTRY(invalidate_cache)
3044 3033 wbinvd
3045 3034 ret
3046 3035 SET_SIZE(invalidate_cache)
3047 3036
3048 3037 #endif /* __lint */
3049 3038
3050 3039 #if defined(__lint)
3051 3040
3052 3041 /*ARGSUSED*/
3053 3042 void
3054 3043 getcregs(struct cregs *crp)
3055 3044 {}
3056 3045
3057 3046 #else /* __lint */
3058 3047
3059 3048 #if defined(__amd64)
3060 3049
3061 3050 ENTRY_NP(getcregs)
3062 3051 #if defined(__xpv)
3063 3052 /*
3064 3053 * Only a few of the hardware control registers or descriptor tables
3065 3054 * are directly accessible to us, so just zero the structure.
3066 3055 *
3067 3056 * XXPV Perhaps it would be helpful for the hypervisor to return
3068 3057 * virtualized versions of these for post-mortem use.
3069 3058 * (Need to reevaluate - perhaps it already does!)
3070 3059 */
3071 3060 pushq %rdi /* save *crp */
3072 3061 movq $CREGSZ, %rsi
3073 3062 call bzero
3074 3063 popq %rdi
3075 3064
3076 3065 /*
3077 3066 * Dump what limited information we can
3078 3067 */
3079 3068 movq %cr0, %rax
3080 3069 movq %rax, CREG_CR0(%rdi) /* cr0 */
3081 3070 movq %cr2, %rax
3082 3071 movq %rax, CREG_CR2(%rdi) /* cr2 */
3083 3072 movq %cr3, %rax
3084 3073 movq %rax, CREG_CR3(%rdi) /* cr3 */
3085 3074 movq %cr4, %rax
3086 3075 movq %rax, CREG_CR4(%rdi) /* cr4 */
3087 3076
3088 3077 #else /* __xpv */
3089 3078
3090 3079 #define GETMSR(r, off, d) \
3091 3080 movl $r, %ecx; \
3092 3081 rdmsr; \
3093 3082 movl %eax, off(d); \
3094 3083 movl %edx, off+4(d)
3095 3084
3096 3085 xorl %eax, %eax
3097 3086 movq %rax, CREG_GDT+8(%rdi)
3098 3087 sgdt CREG_GDT(%rdi) /* 10 bytes */
3099 3088 movq %rax, CREG_IDT+8(%rdi)
3100 3089 sidt CREG_IDT(%rdi) /* 10 bytes */
3101 3090 movq %rax, CREG_LDT(%rdi)
3102 3091 sldt CREG_LDT(%rdi) /* 2 bytes */
3103 3092 movq %rax, CREG_TASKR(%rdi)
3104 3093 str CREG_TASKR(%rdi) /* 2 bytes */
3105 3094 movq %cr0, %rax
3106 3095 movq %rax, CREG_CR0(%rdi) /* cr0 */
3107 3096 movq %cr2, %rax
3108 3097 movq %rax, CREG_CR2(%rdi) /* cr2 */
3109 3098 movq %cr3, %rax
3110 3099 movq %rax, CREG_CR3(%rdi) /* cr3 */
3111 3100 movq %cr4, %rax
3112 3101 movq %rax, CREG_CR4(%rdi) /* cr4 */
3113 3102 movq %cr8, %rax
3114 3103 movq %rax, CREG_CR8(%rdi) /* cr8 */
3115 3104 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3116 3105 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3117 3106 #endif /* __xpv */
3118 3107 ret
3119 3108 SET_SIZE(getcregs)
3120 3109
3121 3110 #undef GETMSR
3122 3111
3123 3112 #elif defined(__i386)
3124 3113
3125 3114 ENTRY_NP(getcregs)
3126 3115 #if defined(__xpv)
3127 3116 /*
3128 3117 * Only a few of the hardware control registers or descriptor tables
3129 3118 * are directly accessible to us, so just zero the structure.
3130 3119 *
3131 3120 * XXPV Perhaps it would be helpful for the hypervisor to return
3132 3121 * virtualized versions of these for post-mortem use.
3133 3122 * (Need to reevaluate - perhaps it already does!)
3134 3123 */
3135 3124 movl 4(%esp), %edx
3136 3125 pushl $CREGSZ
3137 3126 pushl %edx
3138 3127 call bzero
3139 3128 addl $8, %esp
3140 3129 movl 4(%esp), %edx
3141 3130
3142 3131 /*
3143 3132 * Dump what limited information we can
3144 3133 */
3145 3134 movl %cr0, %eax
3146 3135 movl %eax, CREG_CR0(%edx) /* cr0 */
3147 3136 movl %cr2, %eax
3148 3137 movl %eax, CREG_CR2(%edx) /* cr2 */
3149 3138 movl %cr3, %eax
3150 3139 movl %eax, CREG_CR3(%edx) /* cr3 */
3151 3140 movl %cr4, %eax
3152 3141 movl %eax, CREG_CR4(%edx) /* cr4 */
3153 3142
3154 3143 #else /* __xpv */
3155 3144
3156 3145 movl 4(%esp), %edx
3157 3146 movw $0, CREG_GDT+6(%edx)
3158 3147 movw $0, CREG_IDT+6(%edx)
3159 3148 sgdt CREG_GDT(%edx) /* gdt */
3160 3149 sidt CREG_IDT(%edx) /* idt */
3161 3150 sldt CREG_LDT(%edx) /* ldt */
3162 3151 str CREG_TASKR(%edx) /* task */
3163 3152 movl %cr0, %eax
3164 3153 movl %eax, CREG_CR0(%edx) /* cr0 */
3165 3154 movl %cr2, %eax
3166 3155 movl %eax, CREG_CR2(%edx) /* cr2 */
3167 3156 movl %cr3, %eax
3168 3157 movl %eax, CREG_CR3(%edx) /* cr3 */
3169 3158 bt $X86FSET_LARGEPAGE, x86_featureset
3170 3159 jnc .nocr4
3171 3160 movl %cr4, %eax
3172 3161 movl %eax, CREG_CR4(%edx) /* cr4 */
3173 3162 jmp .skip
3174 3163 .nocr4:
3175 3164 movl $0, CREG_CR4(%edx)
3176 3165 .skip:
3177 3166 #endif
3178 3167 ret
3179 3168 SET_SIZE(getcregs)
3180 3169
3181 3170 #endif /* __i386 */
3182 3171 #endif /* __lint */
3183 3172
3184 3173
3185 3174 /*
3186 3175 * A panic trigger is a word which is updated atomically and can only be set
3187 3176 * once. We atomically store 0xDEFACEDD and load the old value. If the
3188 3177 * previous value was 0, we succeed and return 1; otherwise return 0.
3189 3178 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3190 3179 * has its own version of this function to allow it to panic correctly from
3191 3180 * probe context.
3192 3181 */
3193 3182 #if defined(__lint)
3194 3183
3195 3184 /*ARGSUSED*/
3196 3185 int
3197 3186 panic_trigger(int *tp)
3198 3187 { return (0); }
3199 3188
3200 3189 /*ARGSUSED*/
3201 3190 int
3202 3191 dtrace_panic_trigger(int *tp)
3203 3192 { return (0); }
3204 3193
3205 3194 #else /* __lint */
3206 3195
3207 3196 #if defined(__amd64)
3208 3197
3209 3198 ENTRY_NP(panic_trigger)
3210 3199 xorl %eax, %eax
3211 3200 movl $0xdefacedd, %edx
3212 3201 lock
3213 3202 xchgl %edx, (%rdi)
3214 3203 cmpl $0, %edx
3215 3204 je 0f
3216 3205 movl $0, %eax
3217 3206 ret
3218 3207 0: movl $1, %eax
3219 3208 ret
3220 3209 SET_SIZE(panic_trigger)
3221 3210
3222 3211 ENTRY_NP(dtrace_panic_trigger)
3223 3212 xorl %eax, %eax
3224 3213 movl $0xdefacedd, %edx
3225 3214 lock
3226 3215 xchgl %edx, (%rdi)
3227 3216 cmpl $0, %edx
3228 3217 je 0f
3229 3218 movl $0, %eax
3230 3219 ret
3231 3220 0: movl $1, %eax
3232 3221 ret
3233 3222 SET_SIZE(dtrace_panic_trigger)
3234 3223
3235 3224 #elif defined(__i386)
3236 3225
3237 3226 ENTRY_NP(panic_trigger)
3238 3227 movl 4(%esp), %edx / %edx = address of trigger
3239 3228 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3240 3229 lock / assert lock
3241 3230 xchgl %eax, (%edx) / exchange %eax and the trigger
3242 3231 cmpl $0, %eax / if (%eax == 0x0)
3243 3232 je 0f / return (1);
3244 3233 movl $0, %eax / else
3245 3234 ret / return (0);
3246 3235 0: movl $1, %eax
3247 3236 ret
3248 3237 SET_SIZE(panic_trigger)
3249 3238
3250 3239 ENTRY_NP(dtrace_panic_trigger)
3251 3240 movl 4(%esp), %edx / %edx = address of trigger
3252 3241 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3253 3242 lock / assert lock
3254 3243 xchgl %eax, (%edx) / exchange %eax and the trigger
3255 3244 cmpl $0, %eax / if (%eax == 0x0)
3256 3245 je 0f / return (1);
3257 3246 movl $0, %eax / else
3258 3247 ret / return (0);
3259 3248 0: movl $1, %eax
3260 3249 ret
3261 3250 SET_SIZE(dtrace_panic_trigger)
3262 3251
3263 3252 #endif /* __i386 */
3264 3253 #endif /* __lint */
3265 3254
3266 3255 /*
3267 3256 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3268 3257 * into the panic code implemented in panicsys(). vpanic() is responsible
3269 3258 * for passing through the format string and arguments, and constructing a
3270 3259 * regs structure on the stack into which it saves the current register
3271 3260 * values. If we are not dying due to a fatal trap, these registers will
3272 3261 * then be preserved in panicbuf as the current processor state. Before
3273 3262 * invoking panicsys(), vpanic() activates the first panic trigger (see
3274 3263 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3275 3264 * DTrace takes a slightly different panic path if it must panic from probe
3276 3265 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3277 3266 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3278 3267 * branches back into vpanic().
3279 3268 */
3280 3269 #if defined(__lint)
3281 3270
3282 3271 /*ARGSUSED*/
3283 3272 void
3284 3273 vpanic(const char *format, va_list alist)
3285 3274 {}
3286 3275
3287 3276 /*ARGSUSED*/
3288 3277 void
3289 3278 dtrace_vpanic(const char *format, va_list alist)
3290 3279 {}
3291 3280
3292 3281 #else /* __lint */
3293 3282
3294 3283 #if defined(__amd64)
3295 3284
3296 3285 ENTRY_NP(vpanic) /* Initial stack layout: */
3297 3286
3298 3287 pushq %rbp /* | %rip | 0x60 */
3299 3288 movq %rsp, %rbp /* | %rbp | 0x58 */
3300 3289 pushfq /* | rfl | 0x50 */
3301 3290 pushq %r11 /* | %r11 | 0x48 */
3302 3291 pushq %r10 /* | %r10 | 0x40 */
3303 3292 pushq %rbx /* | %rbx | 0x38 */
3304 3293 pushq %rax /* | %rax | 0x30 */
3305 3294 pushq %r9 /* | %r9 | 0x28 */
3306 3295 pushq %r8 /* | %r8 | 0x20 */
3307 3296 pushq %rcx /* | %rcx | 0x18 */
3308 3297 pushq %rdx /* | %rdx | 0x10 */
3309 3298 pushq %rsi /* | %rsi | 0x8 alist */
3310 3299 pushq %rdi /* | %rdi | 0x0 format */
3311 3300
3312 3301 movq %rsp, %rbx /* %rbx = current %rsp */
3313 3302
3314 3303 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3315 3304 call panic_trigger /* %eax = panic_trigger() */
3316 3305
3317 3306 vpanic_common:
3318 3307 /*
3319 3308 * The panic_trigger result is in %eax from the call above, and
3320 3309 * dtrace_panic places it in %eax before branching here.
3321 3310 * The rdmsr instructions that follow below will clobber %eax so
3322 3311 * we stash the panic_trigger result in %r11d.
3323 3312 */
3324 3313 movl %eax, %r11d
3325 3314 cmpl $0, %r11d
3326 3315 je 0f
3327 3316
3328 3317 /*
3329 3318 * If panic_trigger() was successful, we are the first to initiate a
3330 3319 * panic: we now switch to the reserved panic_stack before continuing.
3331 3320 */
3332 3321 leaq panic_stack(%rip), %rsp
3333 3322 addq $PANICSTKSIZE, %rsp
3334 3323 0: subq $REGSIZE, %rsp
3335 3324 /*
3336 3325 * Now that we've got everything set up, store the register values as
3337 3326 * they were when we entered vpanic() to the designated location in
3338 3327 * the regs structure we allocated on the stack.
3339 3328 */
3340 3329 movq 0x0(%rbx), %rcx
3341 3330 movq %rcx, REGOFF_RDI(%rsp)
3342 3331 movq 0x8(%rbx), %rcx
3343 3332 movq %rcx, REGOFF_RSI(%rsp)
3344 3333 movq 0x10(%rbx), %rcx
3345 3334 movq %rcx, REGOFF_RDX(%rsp)
3346 3335 movq 0x18(%rbx), %rcx
3347 3336 movq %rcx, REGOFF_RCX(%rsp)
3348 3337 movq 0x20(%rbx), %rcx
3349 3338
3350 3339 movq %rcx, REGOFF_R8(%rsp)
3351 3340 movq 0x28(%rbx), %rcx
3352 3341 movq %rcx, REGOFF_R9(%rsp)
3353 3342 movq 0x30(%rbx), %rcx
3354 3343 movq %rcx, REGOFF_RAX(%rsp)
3355 3344 movq 0x38(%rbx), %rcx
3356 3345 movq %rcx, REGOFF_RBX(%rsp)
3357 3346 movq 0x58(%rbx), %rcx
3358 3347
3359 3348 movq %rcx, REGOFF_RBP(%rsp)
3360 3349 movq 0x40(%rbx), %rcx
3361 3350 movq %rcx, REGOFF_R10(%rsp)
3362 3351 movq 0x48(%rbx), %rcx
3363 3352 movq %rcx, REGOFF_R11(%rsp)
3364 3353 movq %r12, REGOFF_R12(%rsp)
3365 3354
3366 3355 movq %r13, REGOFF_R13(%rsp)
3367 3356 movq %r14, REGOFF_R14(%rsp)
3368 3357 movq %r15, REGOFF_R15(%rsp)
3369 3358
3370 3359 xorl %ecx, %ecx
3371 3360 movw %ds, %cx
3372 3361 movq %rcx, REGOFF_DS(%rsp)
3373 3362 movw %es, %cx
3374 3363 movq %rcx, REGOFF_ES(%rsp)
3375 3364 movw %fs, %cx
3376 3365 movq %rcx, REGOFF_FS(%rsp)
3377 3366 movw %gs, %cx
3378 3367 movq %rcx, REGOFF_GS(%rsp)
3379 3368
3380 3369 movq $0, REGOFF_TRAPNO(%rsp)
3381 3370
3382 3371 movq $0, REGOFF_ERR(%rsp)
3383 3372 leaq vpanic(%rip), %rcx
3384 3373 movq %rcx, REGOFF_RIP(%rsp)
3385 3374 movw %cs, %cx
3386 3375 movzwq %cx, %rcx
3387 3376 movq %rcx, REGOFF_CS(%rsp)
3388 3377 movq 0x50(%rbx), %rcx
3389 3378 movq %rcx, REGOFF_RFL(%rsp)
3390 3379 movq %rbx, %rcx
3391 3380 addq $0x60, %rcx
3392 3381 movq %rcx, REGOFF_RSP(%rsp)
3393 3382 movw %ss, %cx
3394 3383 movzwq %cx, %rcx
3395 3384 movq %rcx, REGOFF_SS(%rsp)
3396 3385
3397 3386 /*
3398 3387 * panicsys(format, alist, rp, on_panic_stack)
3399 3388 */
3400 3389 movq REGOFF_RDI(%rsp), %rdi /* format */
3401 3390 movq REGOFF_RSI(%rsp), %rsi /* alist */
3402 3391 movq %rsp, %rdx /* struct regs */
3403 3392 movl %r11d, %ecx /* on_panic_stack */
3404 3393 call panicsys
3405 3394 addq $REGSIZE, %rsp
3406 3395 popq %rdi
3407 3396 popq %rsi
3408 3397 popq %rdx
3409 3398 popq %rcx
3410 3399 popq %r8
3411 3400 popq %r9
3412 3401 popq %rax
3413 3402 popq %rbx
3414 3403 popq %r10
3415 3404 popq %r11
3416 3405 popfq
3417 3406 leave
3418 3407 ret
3419 3408 SET_SIZE(vpanic)
3420 3409
3421 3410 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3422 3411
3423 3412 pushq %rbp /* | %rip | 0x60 */
3424 3413 movq %rsp, %rbp /* | %rbp | 0x58 */
3425 3414 pushfq /* | rfl | 0x50 */
3426 3415 pushq %r11 /* | %r11 | 0x48 */
3427 3416 pushq %r10 /* | %r10 | 0x40 */
3428 3417 pushq %rbx /* | %rbx | 0x38 */
3429 3418 pushq %rax /* | %rax | 0x30 */
3430 3419 pushq %r9 /* | %r9 | 0x28 */
3431 3420 pushq %r8 /* | %r8 | 0x20 */
3432 3421 pushq %rcx /* | %rcx | 0x18 */
3433 3422 pushq %rdx /* | %rdx | 0x10 */
3434 3423 pushq %rsi /* | %rsi | 0x8 alist */
3435 3424 pushq %rdi /* | %rdi | 0x0 format */
3436 3425
3437 3426 movq %rsp, %rbx /* %rbx = current %rsp */
3438 3427
3439 3428 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3440 3429 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3441 3430 jmp vpanic_common
3442 3431
3443 3432 SET_SIZE(dtrace_vpanic)
3444 3433
3445 3434 #elif defined(__i386)
3446 3435
3447 3436 ENTRY_NP(vpanic) / Initial stack layout:
3448 3437
3449 3438 pushl %ebp / | %eip | 20
3450 3439 movl %esp, %ebp / | %ebp | 16
3451 3440 pushl %eax / | %eax | 12
3452 3441 pushl %ebx / | %ebx | 8
3453 3442 pushl %ecx / | %ecx | 4
3454 3443 pushl %edx / | %edx | 0
3455 3444
3456 3445 movl %esp, %ebx / %ebx = current stack pointer
3457 3446
3458 3447 lea panic_quiesce, %eax / %eax = &panic_quiesce
3459 3448 pushl %eax / push &panic_quiesce
3460 3449 call panic_trigger / %eax = panic_trigger()
3461 3450 addl $4, %esp / reset stack pointer
3462 3451
3463 3452 vpanic_common:
3464 3453 cmpl $0, %eax / if (%eax == 0)
3465 3454 je 0f / goto 0f;
3466 3455
3467 3456 /*
3468 3457 * If panic_trigger() was successful, we are the first to initiate a
3469 3458 * panic: we now switch to the reserved panic_stack before continuing.
3470 3459 */
3471 3460 lea panic_stack, %esp / %esp = panic_stack
3472 3461 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3473 3462
3474 3463 0: subl $REGSIZE, %esp / allocate struct regs
3475 3464
3476 3465 /*
3477 3466 * Now that we've got everything set up, store the register values as
3478 3467 * they were when we entered vpanic() to the designated location in
3479 3468 * the regs structure we allocated on the stack.
3480 3469 */
3481 3470 #if !defined(__GNUC_AS__)
3482 3471 movw %gs, %edx
3483 3472 movl %edx, REGOFF_GS(%esp)
3484 3473 movw %fs, %edx
3485 3474 movl %edx, REGOFF_FS(%esp)
3486 3475 movw %es, %edx
3487 3476 movl %edx, REGOFF_ES(%esp)
3488 3477 movw %ds, %edx
3489 3478 movl %edx, REGOFF_DS(%esp)
3490 3479 #else /* __GNUC_AS__ */
3491 3480 mov %gs, %edx
3492 3481 mov %edx, REGOFF_GS(%esp)
3493 3482 mov %fs, %edx
3494 3483 mov %edx, REGOFF_FS(%esp)
3495 3484 mov %es, %edx
3496 3485 mov %edx, REGOFF_ES(%esp)
3497 3486 mov %ds, %edx
3498 3487 mov %edx, REGOFF_DS(%esp)
3499 3488 #endif /* __GNUC_AS__ */
3500 3489 movl %edi, REGOFF_EDI(%esp)
3501 3490 movl %esi, REGOFF_ESI(%esp)
3502 3491 movl 16(%ebx), %ecx
3503 3492 movl %ecx, REGOFF_EBP(%esp)
3504 3493 movl %ebx, %ecx
3505 3494 addl $20, %ecx
3506 3495 movl %ecx, REGOFF_ESP(%esp)
3507 3496 movl 8(%ebx), %ecx
3508 3497 movl %ecx, REGOFF_EBX(%esp)
3509 3498 movl 0(%ebx), %ecx
3510 3499 movl %ecx, REGOFF_EDX(%esp)
3511 3500 movl 4(%ebx), %ecx
3512 3501 movl %ecx, REGOFF_ECX(%esp)
3513 3502 movl 12(%ebx), %ecx
3514 3503 movl %ecx, REGOFF_EAX(%esp)
3515 3504 movl $0, REGOFF_TRAPNO(%esp)
3516 3505 movl $0, REGOFF_ERR(%esp)
3517 3506 lea vpanic, %ecx
3518 3507 movl %ecx, REGOFF_EIP(%esp)
3519 3508 #if !defined(__GNUC_AS__)
3520 3509 movw %cs, %edx
3521 3510 #else /* __GNUC_AS__ */
3522 3511 mov %cs, %edx
3523 3512 #endif /* __GNUC_AS__ */
3524 3513 movl %edx, REGOFF_CS(%esp)
3525 3514 pushfl
3526 3515 popl %ecx
3527 3516 #if defined(__xpv)
3528 3517 /*
3529 3518 * Synthesize the PS_IE bit from the event mask bit
3530 3519 */
3531 3520 CURTHREAD(%edx)
3532 3521 KPREEMPT_DISABLE(%edx)
3533 3522 EVENT_MASK_TO_IE(%edx, %ecx)
3534 3523 CURTHREAD(%edx)
3535 3524 KPREEMPT_ENABLE_NOKP(%edx)
3536 3525 #endif
3537 3526 movl %ecx, REGOFF_EFL(%esp)
3538 3527 movl $0, REGOFF_UESP(%esp)
3539 3528 #if !defined(__GNUC_AS__)
3540 3529 movw %ss, %edx
3541 3530 #else /* __GNUC_AS__ */
3542 3531 mov %ss, %edx
3543 3532 #endif /* __GNUC_AS__ */
3544 3533 movl %edx, REGOFF_SS(%esp)
3545 3534
3546 3535 movl %esp, %ecx / %ecx = ®s
3547 3536 pushl %eax / push on_panic_stack
3548 3537 pushl %ecx / push ®s
3549 3538 movl 12(%ebp), %ecx / %ecx = alist
3550 3539 pushl %ecx / push alist
3551 3540 movl 8(%ebp), %ecx / %ecx = format
3552 3541 pushl %ecx / push format
3553 3542 call panicsys / panicsys();
3554 3543 addl $16, %esp / pop arguments
3555 3544
3556 3545 addl $REGSIZE, %esp
3557 3546 popl %edx
3558 3547 popl %ecx
3559 3548 popl %ebx
3560 3549 popl %eax
3561 3550 leave
3562 3551 ret
3563 3552 SET_SIZE(vpanic)
3564 3553
3565 3554 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3566 3555
3567 3556 pushl %ebp / | %eip | 20
3568 3557 movl %esp, %ebp / | %ebp | 16
3569 3558 pushl %eax / | %eax | 12
3570 3559 pushl %ebx / | %ebx | 8
3571 3560 pushl %ecx / | %ecx | 4
3572 3561 pushl %edx / | %edx | 0
3573 3562
3574 3563 movl %esp, %ebx / %ebx = current stack pointer
3575 3564
3576 3565 lea panic_quiesce, %eax / %eax = &panic_quiesce
3577 3566 pushl %eax / push &panic_quiesce
3578 3567 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3579 3568 addl $4, %esp / reset stack pointer
3580 3569 jmp vpanic_common / jump back to common code
3581 3570
3582 3571 SET_SIZE(dtrace_vpanic)
3583 3572
3584 3573 #endif /* __i386 */
3585 3574 #endif /* __lint */
3586 3575
3587 3576 #if defined(__lint)
3588 3577
3589 3578 void
3590 3579 hres_tick(void)
3591 3580 {}
3592 3581
3593 3582 int64_t timedelta;
3594 3583 hrtime_t hrtime_base;
3595 3584
3596 3585 #else /* __lint */
3597 3586
3598 3587 DGDEF3(timedelta, 8, 8)
3599 3588 .long 0, 0
3600 3589
3601 3590 /*
3602 3591 * initialized to a non zero value to make pc_gethrtime()
3603 3592 * work correctly even before clock is initialized
3604 3593 */
3605 3594 DGDEF3(hrtime_base, 8, 8)
3606 3595 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3607 3596
3608 3597 DGDEF3(adj_shift, 4, 4)
3609 3598 .long ADJ_SHIFT
3610 3599
3611 3600 #if defined(__amd64)
3612 3601
3613 3602 ENTRY_NP(hres_tick)
3614 3603 pushq %rbp
3615 3604 movq %rsp, %rbp
3616 3605
3617 3606 /*
3618 3607 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3619 3608 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3620 3609 * At worst, performing this now instead of under CLOCK_LOCK may
3621 3610 * introduce some jitter in pc_gethrestime().
3622 3611 */
3623 3612 call *gethrtimef(%rip)
3624 3613 movq %rax, %r8
3625 3614
3626 3615 leaq hres_lock(%rip), %rax
3627 3616 movb $-1, %dl
3628 3617 .CL1:
3629 3618 xchgb %dl, (%rax)
3630 3619 testb %dl, %dl
3631 3620 jz .CL3 /* got it */
3632 3621 .CL2:
3633 3622 cmpb $0, (%rax) /* possible to get lock? */
3634 3623 pause
3635 3624 jne .CL2
3636 3625 jmp .CL1 /* yes, try again */
3637 3626 .CL3:
3638 3627 /*
3639 3628 * compute the interval since last time hres_tick was called
3640 3629 * and adjust hrtime_base and hrestime accordingly
3641 3630 * hrtime_base is an 8 byte value (in nsec), hrestime is
3642 3631 * a timestruc_t (sec, nsec)
3643 3632 */
3644 3633 leaq hres_last_tick(%rip), %rax
3645 3634 movq %r8, %r11
3646 3635 subq (%rax), %r8
3647 3636 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3648 3637 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3649 3638 /*
3650 3639 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3651 3640 */
3652 3641 movq %r11, (%rax)
3653 3642
3654 3643 call __adj_hrestime
3655 3644
3656 3645 /*
3657 3646 * release the hres_lock
3658 3647 */
3659 3648 incl hres_lock(%rip)
3660 3649 leave
3661 3650 ret
3662 3651 SET_SIZE(hres_tick)
3663 3652
3664 3653 #elif defined(__i386)
3665 3654
3666 3655 ENTRY_NP(hres_tick)
3667 3656 pushl %ebp
3668 3657 movl %esp, %ebp
3669 3658 pushl %esi
3670 3659 pushl %ebx
3671 3660
3672 3661 /*
3673 3662 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3674 3663 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3675 3664 * At worst, performing this now instead of under CLOCK_LOCK may
3676 3665 * introduce some jitter in pc_gethrestime().
3677 3666 */
3678 3667 call *gethrtimef
3679 3668 movl %eax, %ebx
3680 3669 movl %edx, %esi
3681 3670
3682 3671 movl $hres_lock, %eax
3683 3672 movl $-1, %edx
3684 3673 .CL1:
3685 3674 xchgb %dl, (%eax)
3686 3675 testb %dl, %dl
3687 3676 jz .CL3 / got it
3688 3677 .CL2:
3689 3678 cmpb $0, (%eax) / possible to get lock?
3690 3679 pause
3691 3680 jne .CL2
3692 3681 jmp .CL1 / yes, try again
3693 3682 .CL3:
3694 3683 /*
3695 3684 * compute the interval since last time hres_tick was called
3696 3685 * and adjust hrtime_base and hrestime accordingly
3697 3686 * hrtime_base is an 8 byte value (in nsec), hrestime is
3698 3687 * timestruc_t (sec, nsec)
3699 3688 */
3700 3689
3701 3690 lea hres_last_tick, %eax
3702 3691
3703 3692 movl %ebx, %edx
3704 3693 movl %esi, %ecx
3705 3694
3706 3695 subl (%eax), %edx
3707 3696 sbbl 4(%eax), %ecx
3708 3697
3709 3698 addl %edx, hrtime_base / add interval to hrtime_base
3710 3699 adcl %ecx, hrtime_base+4
3711 3700
3712 3701 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3713 3702
3714 3703 /
3715 3704 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3716 3705 /
3717 3706 movl %ebx, (%eax)
3718 3707 movl %esi, 4(%eax)
3719 3708
3720 3709 / get hrestime at this moment. used as base for pc_gethrestime
3721 3710 /
3722 3711 / Apply adjustment, if any
3723 3712 /
3724 3713 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3725 3714 / (max_hres_adj)
3726 3715 /
3727 3716 / void
3728 3717 / adj_hrestime()
3729 3718 / {
3730 3719 / long long adj;
3731 3720 /
3732 3721 / if (hrestime_adj == 0)
3733 3722 / adj = 0;
3734 3723 / else if (hrestime_adj > 0) {
3735 3724 / if (hrestime_adj < HRES_ADJ)
3736 3725 / adj = hrestime_adj;
3737 3726 / else
3738 3727 / adj = HRES_ADJ;
3739 3728 / }
3740 3729 / else {
3741 3730 / if (hrestime_adj < -(HRES_ADJ))
3742 3731 / adj = -(HRES_ADJ);
3743 3732 / else
3744 3733 / adj = hrestime_adj;
3745 3734 / }
3746 3735 /
3747 3736 / timedelta -= adj;
3748 3737 / hrestime_adj = timedelta;
3749 3738 / hrestime.tv_nsec += adj;
3750 3739 /
3751 3740 / while (hrestime.tv_nsec >= NANOSEC) {
3752 3741 / one_sec++;
3753 3742 / hrestime.tv_sec++;
3754 3743 / hrestime.tv_nsec -= NANOSEC;
3755 3744 / }
3756 3745 / }
3757 3746 __adj_hrestime:
3758 3747 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3759 3748 movl hrestime_adj+4, %edx
3760 3749 andl %esi, %esi
3761 3750 jne .CL4 / no
3762 3751 andl %edx, %edx
3763 3752 jne .CL4 / no
3764 3753 subl %ecx, %ecx / yes, adj = 0;
3765 3754 subl %edx, %edx
3766 3755 jmp .CL5
3767 3756 .CL4:
3768 3757 subl %ecx, %ecx
3769 3758 subl %eax, %eax
3770 3759 subl %esi, %ecx
3771 3760 sbbl %edx, %eax
3772 3761 andl %eax, %eax / if (hrestime_adj > 0)
3773 3762 jge .CL6
3774 3763
3775 3764 / In the following comments, HRES_ADJ is used, while in the code
3776 3765 / max_hres_adj is used.
3777 3766 /
3778 3767 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3779 3768 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3780 3769 / on the logical equivalence of:
3781 3770 /
3782 3771 / !(hrestime_adj < HRES_ADJ)
3783 3772 /
3784 3773 / and the two step sequence:
3785 3774 /
3786 3775 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3787 3776 /
3788 3777 / which computes whether or not the least significant 32-bits
3789 3778 / of hrestime_adj is greater than HRES_ADJ, followed by:
3790 3779 /
3791 3780 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3792 3781 /
3793 3782 / which generates a carry whenever step 1 is true or the most
3794 3783 / significant long of the longlong hrestime_adj is non-zero.
3795 3784
3796 3785 movl max_hres_adj, %ecx / hrestime_adj is positive
3797 3786 subl %esi, %ecx
3798 3787 movl %edx, %eax
3799 3788 adcl $-1, %eax
3800 3789 jnc .CL7
3801 3790 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3802 3791 subl %edx, %edx
3803 3792 jmp .CL5
3804 3793
3805 3794 / The following computation is similar to the one above.
3806 3795 /
3807 3796 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3808 3797 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3809 3798 / on the logical equivalence of:
3810 3799 /
3811 3800 / (hrestime_adj > -HRES_ADJ)
3812 3801 /
3813 3802 / and the two step sequence:
3814 3803 /
3815 3804 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3816 3805 /
3817 3806 / which means the least significant 32-bits of hrestime_adj is
3818 3807 / greater than -HRES_ADJ, followed by:
3819 3808 /
3820 3809 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3821 3810 /
3822 3811 / which generates a carry only when step 1 is true and the most
3823 3812 / significant long of the longlong hrestime_adj is -1.
3824 3813
3825 3814 .CL6: / hrestime_adj is negative
3826 3815 movl %esi, %ecx
3827 3816 addl max_hres_adj, %ecx
3828 3817 movl %edx, %eax
3829 3818 adcl $0, %eax
3830 3819 jc .CL7
3831 3820 xor %ecx, %ecx
3832 3821 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3833 3822 movl $-1, %edx
3834 3823 jmp .CL5
3835 3824 .CL7:
3836 3825 movl %esi, %ecx / adj = hrestime_adj;
3837 3826 .CL5:
3838 3827 movl timedelta, %esi
3839 3828 subl %ecx, %esi
3840 3829 movl timedelta+4, %eax
3841 3830 sbbl %edx, %eax
3842 3831 movl %esi, timedelta
3843 3832 movl %eax, timedelta+4 / timedelta -= adj;
3844 3833 movl %esi, hrestime_adj
3845 3834 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3846 3835 addl hrestime+4, %ecx
3847 3836
3848 3837 movl %ecx, %eax / eax = tv_nsec
3849 3838 1:
3850 3839 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3851 3840 jb .CL8 / no
3852 3841 incl one_sec / yes, one_sec++;
3853 3842 incl hrestime / hrestime.tv_sec++;
3854 3843 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3855 3844 jmp 1b / check for more seconds
3856 3845
3857 3846 .CL8:
3858 3847 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3859 3848 incl hres_lock / release the hres_lock
3860 3849
3861 3850 popl %ebx
3862 3851 popl %esi
3863 3852 leave
3864 3853 ret
3865 3854 SET_SIZE(hres_tick)
3866 3855
3867 3856 #endif /* __i386 */
3868 3857 #endif /* __lint */
3869 3858
3870 3859 /*
3871 3860 * void prefetch_smap_w(void *)
3872 3861 *
3873 3862 * Prefetch ahead within a linear list of smap structures.
3874 3863 * Not implemented for ia32. Stub for compatibility.
3875 3864 */
3876 3865
3877 3866 #if defined(__lint)
3878 3867
3879 3868 /*ARGSUSED*/
3880 3869 void prefetch_smap_w(void *smp)
3881 3870 {}
3882 3871
3883 3872 #else /* __lint */
3884 3873
3885 3874 ENTRY(prefetch_smap_w)
3886 3875 rep; ret /* use 2 byte return instruction when branch target */
3887 3876 /* AMD Software Optimization Guide - Section 6.2 */
3888 3877 SET_SIZE(prefetch_smap_w)
3889 3878
3890 3879 #endif /* __lint */
3891 3880
3892 3881 /*
3893 3882 * prefetch_page_r(page_t *)
3894 3883 * issue prefetch instructions for a page_t
3895 3884 */
3896 3885 #if defined(__lint)
3897 3886
3898 3887 /*ARGSUSED*/
3899 3888 void
3900 3889 prefetch_page_r(void *pp)
3901 3890 {}
3902 3891
3903 3892 #else /* __lint */
3904 3893
3905 3894 ENTRY(prefetch_page_r)
3906 3895 rep; ret /* use 2 byte return instruction when branch target */
3907 3896 /* AMD Software Optimization Guide - Section 6.2 */
3908 3897 SET_SIZE(prefetch_page_r)
3909 3898
3910 3899 #endif /* __lint */
3911 3900
3912 3901 #if defined(__lint)
3913 3902
3914 3903 /*ARGSUSED*/
3915 3904 int
3916 3905 bcmp(const void *s1, const void *s2, size_t count)
3917 3906 { return (0); }
3918 3907
3919 3908 #else /* __lint */
3920 3909
3921 3910 #if defined(__amd64)
3922 3911
3923 3912 ENTRY(bcmp)
3924 3913 pushq %rbp
3925 3914 movq %rsp, %rbp
3926 3915 #ifdef DEBUG
3927 3916 testq %rdx,%rdx
3928 3917 je 1f
3929 3918 movq postbootkernelbase(%rip), %r11
3930 3919 cmpq %r11, %rdi
3931 3920 jb 0f
3932 3921 cmpq %r11, %rsi
3933 3922 jnb 1f
3934 3923 0: leaq .bcmp_panic_msg(%rip), %rdi
3935 3924 xorl %eax, %eax
3936 3925 call panic
3937 3926 1:
3938 3927 #endif /* DEBUG */
3939 3928 call memcmp
3940 3929 testl %eax, %eax
3941 3930 setne %dl
3942 3931 leave
3943 3932 movzbl %dl, %eax
3944 3933 ret
3945 3934 SET_SIZE(bcmp)
3946 3935
3947 3936 #elif defined(__i386)
3948 3937
3949 3938 #define ARG_S1 8
3950 3939 #define ARG_S2 12
3951 3940 #define ARG_LENGTH 16
3952 3941
3953 3942 ENTRY(bcmp)
3954 3943 pushl %ebp
3955 3944 movl %esp, %ebp / create new stack frame
3956 3945 #ifdef DEBUG
3957 3946 cmpl $0, ARG_LENGTH(%ebp)
3958 3947 je 1f
3959 3948 movl postbootkernelbase, %eax
3960 3949 cmpl %eax, ARG_S1(%ebp)
3961 3950 jb 0f
3962 3951 cmpl %eax, ARG_S2(%ebp)
3963 3952 jnb 1f
3964 3953 0: pushl $.bcmp_panic_msg
3965 3954 call panic
3966 3955 1:
3967 3956 #endif /* DEBUG */
3968 3957
3969 3958 pushl %edi / save register variable
3970 3959 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3971 3960 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3972 3961 cmpl %eax, %ecx / if the same string
3973 3962 je .equal / goto .equal
3974 3963 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3975 3964 cmpl $4, %edi / if %edi < 4
3976 3965 jb .byte_check / goto .byte_check
3977 3966 .align 4
3978 3967 .word_loop:
3979 3968 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3980 3969 leal -4(%edi), %edi / %edi -= 4
3981 3970 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3982 3971 jne .word_not_equal / if not equal, goto .word_not_equal
3983 3972 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3984 3973 leal 4(%eax), %eax / %eax += 4 (next word)
3985 3974 cmpl $4, %edi / if %edi >= 4
3986 3975 jae .word_loop / goto .word_loop
3987 3976 .byte_check:
3988 3977 cmpl $0, %edi / if %edi == 0
3989 3978 je .equal / goto .equal
3990 3979 jmp .byte_loop / goto .byte_loop (checks in bytes)
3991 3980 .word_not_equal:
3992 3981 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3993 3982 .align 4
3994 3983 .byte_loop:
3995 3984 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3996 3985 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3997 3986 jne .not_equal / if not equal, goto .not_equal
3998 3987 incl %ecx / %ecx++ (next byte)
3999 3988 incl %eax / %eax++ (next byte)
4000 3989 decl %edi / %edi--
4001 3990 jnz .byte_loop / if not zero, goto .byte_loop
4002 3991 .equal:
4003 3992 xorl %eax, %eax / %eax = 0
4004 3993 popl %edi / restore register variable
4005 3994 leave / restore old stack frame
4006 3995 ret / return (NULL)
4007 3996 .align 4
4008 3997 .not_equal:
4009 3998 movl $1, %eax / return 1
4010 3999 popl %edi / restore register variable
4011 4000 leave / restore old stack frame
4012 4001 ret / return (NULL)
4013 4002 SET_SIZE(bcmp)
4014 4003
4015 4004 #endif /* __i386 */
4016 4005
4017 4006 #ifdef DEBUG
4018 4007 .text
4019 4008 .bcmp_panic_msg:
4020 4009 .string "bcmp: arguments below kernelbase"
4021 4010 #endif /* DEBUG */
4022 4011
4023 4012 #endif /* __lint */
4024 4013
4025 4014 #if defined(__lint)
4026 4015
4027 4016 uint_t
4028 4017 bsrw_insn(uint16_t mask)
4029 4018 {
4030 4019 uint_t index = sizeof (mask) * NBBY - 1;
4031 4020
4032 4021 while ((mask & (1 << index)) == 0)
4033 4022 index--;
4034 4023 return (index);
4035 4024 }
4036 4025
4037 4026 #else /* __lint */
4038 4027
4039 4028 #if defined(__amd64)
4040 4029
4041 4030 ENTRY_NP(bsrw_insn)
4042 4031 xorl %eax, %eax
4043 4032 bsrw %di, %ax
4044 4033 ret
4045 4034 SET_SIZE(bsrw_insn)
4046 4035
4047 4036 #elif defined(__i386)
4048 4037
4049 4038 ENTRY_NP(bsrw_insn)
4050 4039 movw 4(%esp), %cx
4051 4040 xorl %eax, %eax
4052 4041 bsrw %cx, %ax
4053 4042 ret
4054 4043 SET_SIZE(bsrw_insn)
4055 4044
4056 4045 #endif /* __i386 */
4057 4046 #endif /* __lint */
4058 4047
4059 4048 #if defined(__lint)
4060 4049
4061 4050 uint_t
4062 4051 atomic_btr32(uint32_t *pending, uint_t pil)
4063 4052 {
4064 4053 return (*pending &= ~(1 << pil));
4065 4054 }
4066 4055
4067 4056 #else /* __lint */
4068 4057
4069 4058 #if defined(__i386)
4070 4059
4071 4060 ENTRY_NP(atomic_btr32)
4072 4061 movl 4(%esp), %ecx
4073 4062 movl 8(%esp), %edx
4074 4063 xorl %eax, %eax
4075 4064 lock
4076 4065 btrl %edx, (%ecx)
4077 4066 setc %al
4078 4067 ret
4079 4068 SET_SIZE(atomic_btr32)
4080 4069
4081 4070 #endif /* __i386 */
4082 4071 #endif /* __lint */
4083 4072
4084 4073 #if defined(__lint)
4085 4074
4086 4075 /*ARGSUSED*/
4087 4076 void
4088 4077 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4089 4078 uint_t arg2)
4090 4079 {}
4091 4080
4092 4081 #else /* __lint */
4093 4082
4094 4083 #if defined(__amd64)
4095 4084
4096 4085 ENTRY_NP(switch_sp_and_call)
4097 4086 pushq %rbp
4098 4087 movq %rsp, %rbp /* set up stack frame */
4099 4088 movq %rdi, %rsp /* switch stack pointer */
4100 4089 movq %rdx, %rdi /* pass func arg 1 */
4101 4090 movq %rsi, %r11 /* save function to call */
4102 4091 movq %rcx, %rsi /* pass func arg 2 */
4103 4092 call *%r11 /* call function */
4104 4093 leave /* restore stack */
4105 4094 ret
4106 4095 SET_SIZE(switch_sp_and_call)
4107 4096
4108 4097 #elif defined(__i386)
4109 4098
4110 4099 ENTRY_NP(switch_sp_and_call)
4111 4100 pushl %ebp
4112 4101 mov %esp, %ebp /* set up stack frame */
4113 4102 movl 8(%ebp), %esp /* switch stack pointer */
4114 4103 pushl 20(%ebp) /* push func arg 2 */
4115 4104 pushl 16(%ebp) /* push func arg 1 */
4116 4105 call *12(%ebp) /* call function */
4117 4106 addl $8, %esp /* pop arguments */
4118 4107 leave /* restore stack */
4119 4108 ret
4120 4109 SET_SIZE(switch_sp_and_call)
4121 4110
4122 4111 #endif /* __i386 */
4123 4112 #endif /* __lint */
4124 4113
4125 4114 #if defined(__lint)
4126 4115
4127 4116 void
4128 4117 kmdb_enter(void)
4129 4118 {}
4130 4119
4131 4120 #else /* __lint */
4132 4121
4133 4122 #if defined(__amd64)
4134 4123
4135 4124 ENTRY_NP(kmdb_enter)
4136 4125 pushq %rbp
4137 4126 movq %rsp, %rbp
4138 4127
4139 4128 /*
4140 4129 * Save flags, do a 'cli' then return the saved flags
4141 4130 */
4142 4131 call intr_clear
4143 4132
4144 4133 int $T_DBGENTR
4145 4134
4146 4135 /*
4147 4136 * Restore the saved flags
4148 4137 */
4149 4138 movq %rax, %rdi
4150 4139 call intr_restore
4151 4140
4152 4141 leave
4153 4142 ret
4154 4143 SET_SIZE(kmdb_enter)
4155 4144
4156 4145 #elif defined(__i386)
4157 4146
4158 4147 ENTRY_NP(kmdb_enter)
4159 4148 pushl %ebp
4160 4149 movl %esp, %ebp
4161 4150
4162 4151 /*
4163 4152 * Save flags, do a 'cli' then return the saved flags
4164 4153 */
4165 4154 call intr_clear
4166 4155
4167 4156 int $T_DBGENTR
4168 4157
4169 4158 /*
4170 4159 * Restore the saved flags
4171 4160 */
4172 4161 pushl %eax
4173 4162 call intr_restore
4174 4163 addl $4, %esp
4175 4164
4176 4165 leave
4177 4166 ret
4178 4167 SET_SIZE(kmdb_enter)
4179 4168
4180 4169 #endif /* __i386 */
4181 4170 #endif /* __lint */
4182 4171
4183 4172 #if defined(__lint)
4184 4173
4185 4174 void
4186 4175 return_instr(void)
4187 4176 {}
4188 4177
4189 4178 #else /* __lint */
4190 4179
4191 4180 ENTRY_NP(return_instr)
4192 4181 rep; ret /* use 2 byte instruction when branch target */
4193 4182 /* AMD Software Optimization Guide - Section 6.2 */
4194 4183 SET_SIZE(return_instr)
4195 4184
4196 4185 #endif /* __lint */
4197 4186
4198 4187 #if defined(__lint)
4199 4188
4200 4189 ulong_t
4201 4190 getflags(void)
4202 4191 {
4203 4192 return (0);
4204 4193 }
4205 4194
4206 4195 #else /* __lint */
4207 4196
4208 4197 #if defined(__amd64)
4209 4198
4210 4199 ENTRY(getflags)
4211 4200 pushfq
4212 4201 popq %rax
4213 4202 #if defined(__xpv)
4214 4203 CURTHREAD(%rdi)
4215 4204 KPREEMPT_DISABLE(%rdi)
4216 4205 /*
4217 4206 * Synthesize the PS_IE bit from the event mask bit
4218 4207 */
4219 4208 CURVCPU(%r11)
4220 4209 andq $_BITNOT(PS_IE), %rax
4221 4210 XEN_TEST_UPCALL_MASK(%r11)
4222 4211 jnz 1f
4223 4212 orq $PS_IE, %rax
4224 4213 1:
4225 4214 KPREEMPT_ENABLE_NOKP(%rdi)
4226 4215 #endif
4227 4216 ret
4228 4217 SET_SIZE(getflags)
4229 4218
4230 4219 #elif defined(__i386)
4231 4220
4232 4221 ENTRY(getflags)
4233 4222 pushfl
4234 4223 popl %eax
4235 4224 #if defined(__xpv)
4236 4225 CURTHREAD(%ecx)
4237 4226 KPREEMPT_DISABLE(%ecx)
4238 4227 /*
4239 4228 * Synthesize the PS_IE bit from the event mask bit
4240 4229 */
4241 4230 CURVCPU(%edx)
4242 4231 andl $_BITNOT(PS_IE), %eax
4243 4232 XEN_TEST_UPCALL_MASK(%edx)
4244 4233 jnz 1f
4245 4234 orl $PS_IE, %eax
4246 4235 1:
4247 4236 KPREEMPT_ENABLE_NOKP(%ecx)
4248 4237 #endif
4249 4238 ret
4250 4239 SET_SIZE(getflags)
4251 4240
4252 4241 #endif /* __i386 */
4253 4242
4254 4243 #endif /* __lint */
4255 4244
4256 4245 #if defined(__lint)
4257 4246
4258 4247 ftrace_icookie_t
4259 4248 ftrace_interrupt_disable(void)
4260 4249 { return (0); }
4261 4250
4262 4251 #else /* __lint */
4263 4252
4264 4253 #if defined(__amd64)
4265 4254
4266 4255 ENTRY(ftrace_interrupt_disable)
4267 4256 pushfq
4268 4257 popq %rax
4269 4258 CLI(%rdx)
4270 4259 ret
4271 4260 SET_SIZE(ftrace_interrupt_disable)
4272 4261
4273 4262 #elif defined(__i386)
4274 4263
4275 4264 ENTRY(ftrace_interrupt_disable)
4276 4265 pushfl
4277 4266 popl %eax
4278 4267 CLI(%edx)
4279 4268 ret
4280 4269 SET_SIZE(ftrace_interrupt_disable)
4281 4270
4282 4271 #endif /* __i386 */
4283 4272 #endif /* __lint */
4284 4273
4285 4274 #if defined(__lint)
4286 4275
4287 4276 /*ARGSUSED*/
4288 4277 void
4289 4278 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4290 4279 {}
4291 4280
4292 4281 #else /* __lint */
4293 4282
4294 4283 #if defined(__amd64)
4295 4284
4296 4285 ENTRY(ftrace_interrupt_enable)
4297 4286 pushq %rdi
4298 4287 popfq
4299 4288 ret
4300 4289 SET_SIZE(ftrace_interrupt_enable)
4301 4290
4302 4291 #elif defined(__i386)
4303 4292
4304 4293 ENTRY(ftrace_interrupt_enable)
4305 4294 movl 4(%esp), %eax
4306 4295 pushl %eax
4307 4296 popfl
4308 4297 ret
4309 4298 SET_SIZE(ftrace_interrupt_enable)
4310 4299
4311 4300 #endif /* __i386 */
4312 4301 #endif /* __lint */
4313 4302
4314 4303 #if defined (__lint)
4315 4304
4316 4305 /*ARGSUSED*/
4317 4306 void
4318 4307 clflush_insn(caddr_t addr)
4319 4308 {}
4320 4309
4321 4310 #else /* __lint */
4322 4311
4323 4312 #if defined (__amd64)
4324 4313 ENTRY(clflush_insn)
4325 4314 clflush (%rdi)
4326 4315 ret
4327 4316 SET_SIZE(clflush_insn)
4328 4317 #elif defined (__i386)
4329 4318 ENTRY(clflush_insn)
4330 4319 movl 4(%esp), %eax
4331 4320 clflush (%eax)
4332 4321 ret
4333 4322 SET_SIZE(clflush_insn)
4334 4323
4335 4324 #endif /* __i386 */
4336 4325 #endif /* __lint */
4337 4326
4338 4327 #if defined (__lint)
4339 4328 /*ARGSUSED*/
4340 4329 void
4341 4330 mfence_insn(void)
4342 4331 {}
4343 4332
4344 4333 #else /* __lint */
4345 4334
4346 4335 #if defined (__amd64)
4347 4336 ENTRY(mfence_insn)
4348 4337 mfence
4349 4338 ret
4350 4339 SET_SIZE(mfence_insn)
4351 4340 #elif defined (__i386)
4352 4341 ENTRY(mfence_insn)
4353 4342 mfence
4354 4343 ret
4355 4344 SET_SIZE(mfence_insn)
4356 4345
4357 4346 #endif /* __i386 */
4358 4347 #endif /* __lint */
4359 4348
4360 4349 /*
4361 4350 * VMware implements an I/O port that programs can query to detect if software
4362 4351 * is running in a VMware hypervisor. This hypervisor port behaves differently
4363 4352 * depending on magic values in certain registers and modifies some registers
4364 4353 * as a side effect.
4365 4354 *
4366 4355 * References: http://kb.vmware.com/kb/1009458
4367 4356 */
4368 4357
4369 4358 #if defined(__lint)
4370 4359
4371 4360 /* ARGSUSED */
4372 4361 void
4373 4362 vmware_port(int cmd, uint32_t *regs) { return; }
4374 4363
4375 4364 #else
4376 4365
4377 4366 #if defined(__amd64)
4378 4367
4379 4368 ENTRY(vmware_port)
4380 4369 pushq %rbx
4381 4370 movl $VMWARE_HVMAGIC, %eax
4382 4371 movl $0xffffffff, %ebx
4383 4372 movl %edi, %ecx
4384 4373 movl $VMWARE_HVPORT, %edx
4385 4374 inl (%dx)
4386 4375 movl %eax, (%rsi)
4387 4376 movl %ebx, 4(%rsi)
4388 4377 movl %ecx, 8(%rsi)
4389 4378 movl %edx, 12(%rsi)
4390 4379 popq %rbx
4391 4380 ret
4392 4381 SET_SIZE(vmware_port)
4393 4382
4394 4383 #elif defined(__i386)
4395 4384
4396 4385 ENTRY(vmware_port)
4397 4386 pushl %ebx
4398 4387 pushl %esi
4399 4388 movl $VMWARE_HVMAGIC, %eax
4400 4389 movl $0xffffffff, %ebx
4401 4390 movl 12(%esp), %ecx
4402 4391 movl $VMWARE_HVPORT, %edx
4403 4392 inl (%dx)
4404 4393 movl 16(%esp), %esi
4405 4394 movl %eax, (%esi)
4406 4395 movl %ebx, 4(%esi)
4407 4396 movl %ecx, 8(%esi)
4408 4397 movl %edx, 12(%esi)
4409 4398 popl %esi
4410 4399 popl %ebx
4411 4400 ret
4412 4401 SET_SIZE(vmware_port)
4413 4402
4414 4403 #endif /* __i386 */
4415 4404 #endif /* __lint */
|
↓ open down ↓ |
3946 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX