Print this page
11787 Kernel needs to be built with retpolines
11788 Kernel needs to generally use RSB stuffing
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: John Levon <john.levon@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/i86_subr.s
+++ new/usr/src/uts/intel/ia32/ml/i86_subr.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 26 * Copyright 2019 Joyent, Inc.
27 27 */
28 28
29 29 /*
30 30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 32 * All Rights Reserved
33 33 */
34 34
35 35 /*
36 36 * Copyright (c) 2009, Intel Corporation.
37 37 * All rights reserved.
38 38 */
39 39
40 40 /*
41 41 * General assembly language routines.
42 42 * It is the intent of this file to contain routines that are
43 43 * independent of the specific kernel architecture, and those that are
44 44 * common across kernel architectures.
45 45 * As architectures diverge, and implementations of specific
46 46 * architecture-dependent routines change, the routines should be moved
47 47 * from this file into the respective ../`arch -k`/subr.s file.
48 48 */
49 49
50 50 #include <sys/asm_linkage.h>
51 51 #include <sys/asm_misc.h>
52 52 #include <sys/panic.h>
53 53 #include <sys/ontrap.h>
54 54 #include <sys/regset.h>
55 55 #include <sys/privregs.h>
56 56 #include <sys/reboot.h>
57 57 #include <sys/psw.h>
58 58 #include <sys/x86_archext.h>
59 59
60 60 #if defined(__lint)
61 61 #include <sys/types.h>
62 62 #include <sys/systm.h>
63 63 #include <sys/thread.h>
64 64 #include <sys/archsystm.h>
65 65 #include <sys/byteorder.h>
66 66 #include <sys/dtrace.h>
67 67 #include <sys/ftrace.h>
68 68 #else /* __lint */
69 69 #include "assym.h"
70 70 #endif /* __lint */
71 71 #include <sys/dditypes.h>
72 72
73 73 /*
74 74 * on_fault()
75 75 *
76 76 * Catch lofault faults. Like setjmp except it returns one
77 77 * if code following causes uncorrectable fault. Turned off
78 78 * by calling no_fault(). Note that while under on_fault(),
79 79 * SMAP is disabled. For more information see
80 80 * uts/intel/ia32/ml/copy.s.
81 81 */
82 82
83 83 #if defined(__lint)
84 84
85 85 /* ARGSUSED */
86 86 int
87 87 on_fault(label_t *ljb)
88 88 { return (0); }
89 89
90 90 void
91 91 no_fault(void)
92 92 {}
93 93
94 94 #else /* __lint */
95 95
96 96 #if defined(__amd64)
97 97
98 98 ENTRY(on_fault)
99 99 movq %gs:CPU_THREAD, %rsi
100 100 leaq catch_fault(%rip), %rdx
101 101 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
102 102 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
103 103 call smap_disable /* allow user accesses */
104 104 jmp setjmp /* let setjmp do the rest */
105 105
106 106 catch_fault:
107 107 movq %gs:CPU_THREAD, %rsi
108 108 movq T_ONFAULT(%rsi), %rdi /* address of save area */
109 109 xorl %eax, %eax
110 110 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
111 111 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
112 112 call smap_enable /* disallow user accesses */
113 113 jmp longjmp /* let longjmp do the rest */
114 114 SET_SIZE(on_fault)
115 115
116 116 ENTRY(no_fault)
117 117 movq %gs:CPU_THREAD, %rsi
118 118 xorl %eax, %eax
119 119 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
120 120 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
121 121 call smap_enable /* disallow user accesses */
122 122 ret
123 123 SET_SIZE(no_fault)
124 124
125 125 #elif defined(__i386)
126 126
127 127 ENTRY(on_fault)
128 128 movl %gs:CPU_THREAD, %edx
129 129 movl 4(%esp), %eax /* jumpbuf address */
130 130 leal catch_fault, %ecx
131 131 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
132 132 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
133 133 jmp setjmp /* let setjmp do the rest */
134 134
135 135 catch_fault:
136 136 movl %gs:CPU_THREAD, %edx
137 137 xorl %eax, %eax
138 138 movl T_ONFAULT(%edx), %ecx /* address of save area */
139 139 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
140 140 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
141 141 pushl %ecx
142 142 call longjmp /* let longjmp do the rest */
143 143 SET_SIZE(on_fault)
144 144
145 145 ENTRY(no_fault)
146 146 movl %gs:CPU_THREAD, %edx
147 147 xorl %eax, %eax
148 148 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
149 149 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
150 150 ret
151 151 SET_SIZE(no_fault)
152 152
153 153 #endif /* __i386 */
154 154 #endif /* __lint */
155 155
156 156 /*
157 157 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
158 158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 159 */
160 160
161 161 #if defined(lint)
162 162
163 163 void
164 164 on_trap_trampoline(void)
165 165 {}
166 166
167 167 #else /* __lint */
168 168
169 169 #if defined(__amd64)
170 170
171 171 ENTRY(on_trap_trampoline)
172 172 movq %gs:CPU_THREAD, %rsi
173 173 movq T_ONTRAP(%rsi), %rdi
174 174 addq $OT_JMPBUF, %rdi
175 175 jmp longjmp
176 176 SET_SIZE(on_trap_trampoline)
177 177
178 178 #elif defined(__i386)
179 179
180 180 ENTRY(on_trap_trampoline)
181 181 movl %gs:CPU_THREAD, %eax
182 182 movl T_ONTRAP(%eax), %eax
183 183 addl $OT_JMPBUF, %eax
184 184 pushl %eax
185 185 call longjmp
186 186 SET_SIZE(on_trap_trampoline)
187 187
188 188 #endif /* __i386 */
189 189 #endif /* __lint */
190 190
191 191 /*
192 192 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
193 193 * more information about the on_trap() mechanism. If the on_trap_data is the
194 194 * same as the topmost stack element, we just modify that element.
195 195 */
196 196 #if defined(lint)
197 197
198 198 /*ARGSUSED*/
199 199 int
200 200 on_trap(on_trap_data_t *otp, uint_t prot)
201 201 { return (0); }
202 202
203 203 #else /* __lint */
204 204
205 205 #if defined(__amd64)
206 206
207 207 ENTRY(on_trap)
208 208 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
209 209 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
210 210 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
211 211 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
212 212 xorl %ecx, %ecx
213 213 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
214 214 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
215 215 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
216 216 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
217 217 cmpq %rdi, %rcx /* if (otp == %rcx) */
218 218 je 0f /* don't modify t_ontrap */
219 219
220 220 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
221 221 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
222 222
223 223 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
224 224 jmp setjmp
225 225 SET_SIZE(on_trap)
226 226
227 227 #elif defined(__i386)
228 228
229 229 ENTRY(on_trap)
230 230 movl 4(%esp), %eax /* %eax = otp */
231 231 movl 8(%esp), %edx /* %edx = prot */
232 232
233 233 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
234 234 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
235 235 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
236 236 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
237 237 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
238 238 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
239 239 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
240 240 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
241 241 cmpl %eax, %ecx /* if (otp == %ecx) */
242 242 je 0f /* don't modify t_ontrap */
243 243
244 244 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
245 245 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
246 246
247 247 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
248 248 movl %eax, 4(%esp) /* put %eax back on the stack */
249 249 jmp setjmp /* let setjmp do the rest */
250 250 SET_SIZE(on_trap)
251 251
252 252 #endif /* __i386 */
253 253 #endif /* __lint */
254 254
255 255 /*
256 256 * Setjmp and longjmp implement non-local gotos using state vectors
257 257 * type label_t.
258 258 */
259 259
260 260 #if defined(__lint)
261 261
262 262 /* ARGSUSED */
263 263 int
264 264 setjmp(label_t *lp)
265 265 { return (0); }
266 266
267 267 /* ARGSUSED */
268 268 void
269 269 longjmp(label_t *lp)
270 270 {}
271 271
272 272 #else /* __lint */
273 273
274 274 #if LABEL_PC != 0
275 275 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276 276 #endif /* LABEL_PC != 0 */
277 277
278 278 #if defined(__amd64)
279 279
280 280 ENTRY(setjmp)
281 281 movq %rsp, LABEL_SP(%rdi)
282 282 movq %rbp, LABEL_RBP(%rdi)
283 283 movq %rbx, LABEL_RBX(%rdi)
284 284 movq %r12, LABEL_R12(%rdi)
285 285 movq %r13, LABEL_R13(%rdi)
286 286 movq %r14, LABEL_R14(%rdi)
287 287 movq %r15, LABEL_R15(%rdi)
288 288 movq (%rsp), %rdx /* return address */
289 289 movq %rdx, (%rdi) /* LABEL_PC is 0 */
290 290 xorl %eax, %eax /* return 0 */
291 291 ret
292 292 SET_SIZE(setjmp)
293 293
294 294 ENTRY(longjmp)
295 295 movq LABEL_SP(%rdi), %rsp
296 296 movq LABEL_RBP(%rdi), %rbp
297 297 movq LABEL_RBX(%rdi), %rbx
298 298 movq LABEL_R12(%rdi), %r12
299 299 movq LABEL_R13(%rdi), %r13
300 300 movq LABEL_R14(%rdi), %r14
301 301 movq LABEL_R15(%rdi), %r15
302 302 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
303 303 movq %rdx, (%rsp)
304 304 xorl %eax, %eax
305 305 incl %eax /* return 1 */
306 306 ret
307 307 SET_SIZE(longjmp)
308 308
309 309 #elif defined(__i386)
310 310
311 311 ENTRY(setjmp)
312 312 movl 4(%esp), %edx /* address of save area */
313 313 movl %ebp, LABEL_EBP(%edx)
314 314 movl %ebx, LABEL_EBX(%edx)
315 315 movl %esi, LABEL_ESI(%edx)
316 316 movl %edi, LABEL_EDI(%edx)
317 317 movl %esp, 4(%edx)
318 318 movl (%esp), %ecx /* %eip (return address) */
319 319 movl %ecx, (%edx) /* LABEL_PC is 0 */
320 320 subl %eax, %eax /* return 0 */
321 321 ret
322 322 SET_SIZE(setjmp)
323 323
324 324 ENTRY(longjmp)
325 325 movl 4(%esp), %edx /* address of save area */
326 326 movl LABEL_EBP(%edx), %ebp
327 327 movl LABEL_EBX(%edx), %ebx
328 328 movl LABEL_ESI(%edx), %esi
329 329 movl LABEL_EDI(%edx), %edi
330 330 movl 4(%edx), %esp
331 331 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
332 332 movl $1, %eax
333 333 addl $4, %esp /* pop ret adr */
334 334 jmp *%ecx /* indirect */
335 335 SET_SIZE(longjmp)
336 336
337 337 #endif /* __i386 */
338 338 #endif /* __lint */
339 339
340 340 /*
341 341 * if a() calls b() calls caller(),
342 342 * caller() returns return address in a().
343 343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 344 * sequence.)
345 345 */
346 346
347 347 #if defined(__lint)
348 348
349 349 caddr_t
350 350 caller(void)
351 351 { return (0); }
352 352
353 353 #else /* __lint */
354 354
355 355 #if defined(__amd64)
356 356
357 357 ENTRY(caller)
358 358 movq 8(%rbp), %rax /* b()'s return pc, in a() */
359 359 ret
360 360 SET_SIZE(caller)
361 361
362 362 #elif defined(__i386)
363 363
364 364 ENTRY(caller)
365 365 movl 4(%ebp), %eax /* b()'s return pc, in a() */
366 366 ret
367 367 SET_SIZE(caller)
368 368
369 369 #endif /* __i386 */
370 370 #endif /* __lint */
371 371
372 372 /*
373 373 * if a() calls callee(), callee() returns the
374 374 * return address in a();
375 375 */
376 376
377 377 #if defined(__lint)
378 378
379 379 caddr_t
380 380 callee(void)
381 381 { return (0); }
382 382
383 383 #else /* __lint */
384 384
385 385 #if defined(__amd64)
386 386
387 387 ENTRY(callee)
388 388 movq (%rsp), %rax /* callee()'s return pc, in a() */
389 389 ret
390 390 SET_SIZE(callee)
391 391
392 392 #elif defined(__i386)
393 393
394 394 ENTRY(callee)
395 395 movl (%esp), %eax /* callee()'s return pc, in a() */
396 396 ret
397 397 SET_SIZE(callee)
398 398
399 399 #endif /* __i386 */
400 400 #endif /* __lint */
401 401
402 402 /*
403 403 * return the current frame pointer
404 404 */
405 405
406 406 #if defined(__lint)
407 407
408 408 greg_t
409 409 getfp(void)
410 410 { return (0); }
411 411
412 412 #else /* __lint */
413 413
414 414 #if defined(__amd64)
415 415
416 416 ENTRY(getfp)
417 417 movq %rbp, %rax
418 418 ret
419 419 SET_SIZE(getfp)
420 420
421 421 #elif defined(__i386)
422 422
423 423 ENTRY(getfp)
424 424 movl %ebp, %eax
425 425 ret
426 426 SET_SIZE(getfp)
427 427
428 428 #endif /* __i386 */
429 429 #endif /* __lint */
430 430
431 431 /*
432 432 * Invalidate a single page table entry in the TLB
433 433 */
434 434
435 435 #if defined(__lint)
436 436
437 437 /* ARGSUSED */
438 438 void
439 439 mmu_invlpg(caddr_t m)
440 440 {}
441 441
442 442 #else /* __lint */
443 443
444 444 ENTRY(mmu_invlpg)
445 445 invlpg (%rdi)
446 446 ret
447 447 SET_SIZE(mmu_invlpg)
448 448
449 449 #endif /* __lint */
450 450
451 451
452 452 /*
453 453 * Get/Set the value of various control registers
454 454 */
455 455
456 456 #if defined(__lint)
457 457
458 458 ulong_t
459 459 getcr0(void)
460 460 { return (0); }
461 461
462 462 /* ARGSUSED */
463 463 void
464 464 setcr0(ulong_t value)
465 465 {}
466 466
467 467 ulong_t
468 468 getcr2(void)
469 469 { return (0); }
470 470
471 471 ulong_t
472 472 getcr3(void)
473 473 { return (0); }
474 474
475 475 #if !defined(__xpv)
476 476 /* ARGSUSED */
477 477 void
478 478 setcr3(ulong_t val)
479 479 {}
480 480
481 481 void
482 482 reload_cr3(void)
483 483 {}
484 484 #endif
485 485
486 486 ulong_t
487 487 getcr4(void)
488 488 { return (0); }
489 489
490 490 /* ARGSUSED */
491 491 void
492 492 setcr4(ulong_t val)
493 493 {}
494 494
495 495 #if defined(__amd64)
496 496
497 497 ulong_t
498 498 getcr8(void)
499 499 { return (0); }
500 500
501 501 /* ARGSUSED */
502 502 void
503 503 setcr8(ulong_t val)
504 504 {}
505 505
506 506 #endif /* __amd64 */
507 507
508 508 #else /* __lint */
509 509
510 510 #if defined(__amd64)
511 511
512 512 ENTRY(getcr0)
513 513 movq %cr0, %rax
514 514 ret
515 515 SET_SIZE(getcr0)
516 516
517 517 ENTRY(setcr0)
518 518 movq %rdi, %cr0
519 519 ret
520 520 SET_SIZE(setcr0)
521 521
522 522 ENTRY(getcr2)
523 523 #if defined(__xpv)
524 524 movq %gs:CPU_VCPU_INFO, %rax
525 525 movq VCPU_INFO_ARCH_CR2(%rax), %rax
526 526 #else
527 527 movq %cr2, %rax
528 528 #endif
529 529 ret
530 530 SET_SIZE(getcr2)
531 531
532 532 ENTRY(getcr3)
533 533 movq %cr3, %rax
534 534 ret
535 535 SET_SIZE(getcr3)
536 536
537 537 #if !defined(__xpv)
538 538
539 539 ENTRY(setcr3)
540 540 movq %rdi, %cr3
541 541 ret
542 542 SET_SIZE(setcr3)
543 543
544 544 ENTRY(reload_cr3)
545 545 movq %cr3, %rdi
546 546 movq %rdi, %cr3
547 547 ret
548 548 SET_SIZE(reload_cr3)
549 549
550 550 #endif /* __xpv */
551 551
552 552 ENTRY(getcr4)
553 553 movq %cr4, %rax
554 554 ret
555 555 SET_SIZE(getcr4)
556 556
557 557 ENTRY(setcr4)
558 558 movq %rdi, %cr4
559 559 ret
560 560 SET_SIZE(setcr4)
561 561
562 562 ENTRY(getcr8)
563 563 movq %cr8, %rax
564 564 ret
565 565 SET_SIZE(getcr8)
566 566
567 567 ENTRY(setcr8)
568 568 movq %rdi, %cr8
569 569 ret
570 570 SET_SIZE(setcr8)
571 571
572 572 #elif defined(__i386)
573 573
574 574 ENTRY(getcr0)
575 575 movl %cr0, %eax
576 576 ret
577 577 SET_SIZE(getcr0)
578 578
579 579 ENTRY(setcr0)
580 580 movl 4(%esp), %eax
581 581 movl %eax, %cr0
582 582 ret
583 583 SET_SIZE(setcr0)
584 584
585 585 /*
586 586 * "lock mov %cr0" is used on processors which indicate it is
587 587 * supported via CPUID. Normally the 32 bit TPR is accessed via
588 588 * the local APIC.
589 589 */
590 590 ENTRY(getcr8)
591 591 lock
592 592 movl %cr0, %eax
593 593 ret
594 594 SET_SIZE(getcr8)
595 595
596 596 ENTRY(setcr8)
597 597 movl 4(%esp), %eax
598 598 lock
599 599 movl %eax, %cr0
600 600 ret
601 601 SET_SIZE(setcr8)
602 602
603 603 ENTRY(getcr2)
604 604 #if defined(__xpv)
605 605 movl %gs:CPU_VCPU_INFO, %eax
606 606 movl VCPU_INFO_ARCH_CR2(%eax), %eax
607 607 #else
608 608 movl %cr2, %eax
609 609 #endif
610 610 ret
611 611 SET_SIZE(getcr2)
612 612
613 613 ENTRY(getcr3)
614 614 movl %cr3, %eax
615 615 ret
616 616 SET_SIZE(getcr3)
617 617
618 618 #if !defined(__xpv)
619 619
620 620 ENTRY(setcr3)
621 621 movl 4(%esp), %eax
622 622 movl %eax, %cr3
623 623 ret
624 624 SET_SIZE(setcr3)
625 625
626 626 ENTRY(reload_cr3)
627 627 movl %cr3, %eax
628 628 movl %eax, %cr3
629 629 ret
630 630 SET_SIZE(reload_cr3)
631 631
632 632 #endif /* __xpv */
633 633
634 634 ENTRY(getcr4)
635 635 movl %cr4, %eax
636 636 ret
637 637 SET_SIZE(getcr4)
638 638
639 639 ENTRY(setcr4)
640 640 movl 4(%esp), %eax
641 641 movl %eax, %cr4
642 642 ret
643 643 SET_SIZE(setcr4)
644 644
645 645 #endif /* __i386 */
646 646 #endif /* __lint */
647 647
648 648 #if defined(__lint)
649 649
650 650 /*ARGSUSED*/
651 651 uint32_t
652 652 __cpuid_insn(struct cpuid_regs *regs)
653 653 { return (0); }
654 654
655 655 #else /* __lint */
656 656
657 657 #if defined(__amd64)
658 658
659 659 ENTRY(__cpuid_insn)
660 660 movq %rbx, %r8
661 661 movq %rcx, %r9
662 662 movq %rdx, %r11
663 663 movl (%rdi), %eax /* %eax = regs->cp_eax */
664 664 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
665 665 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
666 666 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
667 667 cpuid
668 668 movl %eax, (%rdi) /* regs->cp_eax = %eax */
669 669 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
670 670 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
671 671 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
672 672 movq %r8, %rbx
673 673 movq %r9, %rcx
674 674 movq %r11, %rdx
675 675 ret
676 676 SET_SIZE(__cpuid_insn)
677 677
678 678 #elif defined(__i386)
679 679
680 680 ENTRY(__cpuid_insn)
681 681 pushl %ebp
682 682 movl 0x8(%esp), %ebp /* %ebp = regs */
683 683 pushl %ebx
684 684 pushl %ecx
685 685 pushl %edx
686 686 movl (%ebp), %eax /* %eax = regs->cp_eax */
687 687 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
688 688 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
689 689 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
690 690 cpuid
691 691 movl %eax, (%ebp) /* regs->cp_eax = %eax */
692 692 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
693 693 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
694 694 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
695 695 popl %edx
696 696 popl %ecx
697 697 popl %ebx
698 698 popl %ebp
699 699 ret
700 700 SET_SIZE(__cpuid_insn)
701 701
702 702 #endif /* __i386 */
703 703 #endif /* __lint */
704 704
705 705 #if defined(__lint)
706 706
707 707 /*ARGSUSED*/
708 708 void
709 709 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
710 710 {}
711 711
712 712 #else /* __lint */
713 713
714 714 #if defined(__amd64)
715 715
716 716 ENTRY_NP(i86_monitor)
717 717 pushq %rbp
718 718 movq %rsp, %rbp
719 719 movq %rdi, %rax /* addr */
720 720 movq %rsi, %rcx /* extensions */
721 721 /* rdx contains input arg3: hints */
722 722 clflush (%rax)
723 723 .byte 0x0f, 0x01, 0xc8 /* monitor */
724 724 leave
725 725 ret
726 726 SET_SIZE(i86_monitor)
727 727
728 728 #elif defined(__i386)
729 729
730 730 ENTRY_NP(i86_monitor)
731 731 pushl %ebp
732 732 movl %esp, %ebp
733 733 movl 0x8(%ebp),%eax /* addr */
734 734 movl 0xc(%ebp),%ecx /* extensions */
735 735 movl 0x10(%ebp),%edx /* hints */
736 736 clflush (%eax)
737 737 .byte 0x0f, 0x01, 0xc8 /* monitor */
738 738 leave
739 739 ret
740 740 SET_SIZE(i86_monitor)
741 741
742 742 #endif /* __i386 */
743 743 #endif /* __lint */
744 744
745 745 #if defined(__lint)
746 746
747 747 /*ARGSUSED*/
|
↓ open down ↓ |
747 lines elided |
↑ open up ↑ |
748 748 void
749 749 i86_mwait(uint32_t data, uint32_t extensions)
750 750 {}
751 751
752 752 #else /* __lint */
753 753
754 754 #if defined(__amd64)
755 755
756 756 ENTRY_NP(i86_mwait)
757 757 pushq %rbp
758 - call *x86_md_clear
758 + call x86_md_clear
759 759 movq %rsp, %rbp
760 760 movq %rdi, %rax /* data */
761 761 movq %rsi, %rcx /* extensions */
762 762 .byte 0x0f, 0x01, 0xc9 /* mwait */
763 763 leave
764 764 ret
765 765 SET_SIZE(i86_mwait)
766 766
767 767 #elif defined(__i386)
768 768
769 769 ENTRY_NP(i86_mwait)
770 770 pushl %ebp
771 771 movl %esp, %ebp
772 772 movl 0x8(%ebp),%eax /* data */
773 773 movl 0xc(%ebp),%ecx /* extensions */
774 774 .byte 0x0f, 0x01, 0xc9 /* mwait */
775 775 leave
776 776 ret
777 777 SET_SIZE(i86_mwait)
778 778
779 779 #endif /* __i386 */
780 780 #endif /* __lint */
781 781
782 782 #if defined(__xpv)
783 783 /*
784 784 * Defined in C
785 785 */
786 786 #else
787 787
788 788 #if defined(__lint)
789 789
790 790 hrtime_t
791 791 tsc_read(void)
792 792 {
793 793 return (0);
794 794 }
795 795
796 796 #else /* __lint */
797 797
798 798 #if defined(__amd64)
799 799
800 800 ENTRY_NP(tsc_read)
801 801 movq %rbx, %r11
802 802 movl $0, %eax
803 803 cpuid
804 804 rdtsc
805 805 movq %r11, %rbx
806 806 shlq $32, %rdx
807 807 orq %rdx, %rax
808 808 ret
809 809 .globl _tsc_mfence_start
810 810 _tsc_mfence_start:
811 811 mfence
812 812 rdtsc
813 813 shlq $32, %rdx
814 814 orq %rdx, %rax
815 815 ret
816 816 .globl _tsc_mfence_end
817 817 _tsc_mfence_end:
818 818 .globl _tscp_start
819 819 _tscp_start:
820 820 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
821 821 shlq $32, %rdx
822 822 orq %rdx, %rax
823 823 ret
824 824 .globl _tscp_end
825 825 _tscp_end:
826 826 .globl _no_rdtsc_start
827 827 _no_rdtsc_start:
828 828 xorl %edx, %edx
829 829 xorl %eax, %eax
830 830 ret
831 831 .globl _no_rdtsc_end
832 832 _no_rdtsc_end:
833 833 .globl _tsc_lfence_start
834 834 _tsc_lfence_start:
835 835 lfence
836 836 rdtsc
837 837 shlq $32, %rdx
838 838 orq %rdx, %rax
839 839 ret
840 840 .globl _tsc_lfence_end
841 841 _tsc_lfence_end:
842 842 SET_SIZE(tsc_read)
843 843
844 844 #else /* __i386 */
845 845
846 846 ENTRY_NP(tsc_read)
847 847 pushl %ebx
848 848 movl $0, %eax
849 849 cpuid
850 850 rdtsc
851 851 popl %ebx
852 852 ret
853 853 .globl _tsc_mfence_start
854 854 _tsc_mfence_start:
855 855 mfence
856 856 rdtsc
857 857 ret
858 858 .globl _tsc_mfence_end
859 859 _tsc_mfence_end:
860 860 .globl _tscp_start
861 861 _tscp_start:
862 862 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
863 863 ret
864 864 .globl _tscp_end
865 865 _tscp_end:
866 866 .globl _no_rdtsc_start
867 867 _no_rdtsc_start:
868 868 xorl %edx, %edx
869 869 xorl %eax, %eax
870 870 ret
871 871 .globl _no_rdtsc_end
872 872 _no_rdtsc_end:
873 873 .globl _tsc_lfence_start
874 874 _tsc_lfence_start:
875 875 lfence
876 876 rdtsc
877 877 ret
878 878 .globl _tsc_lfence_end
879 879 _tsc_lfence_end:
880 880 SET_SIZE(tsc_read)
881 881
882 882 #endif /* __i386 */
883 883
884 884 #endif /* __lint */
885 885
886 886
887 887 #endif /* __xpv */
888 888
889 889 #ifdef __lint
890 890 /*
891 891 * Do not use this function for obtaining clock tick. This
892 892 * is called by callers who do not need to have a guarenteed
893 893 * correct tick value. The proper routine to use is tsc_read().
894 894 */
895 895 u_longlong_t
896 896 randtick(void)
897 897 {
898 898 return (0);
899 899 }
900 900 #else
901 901 #if defined(__amd64)
902 902 ENTRY_NP(randtick)
903 903 rdtsc
904 904 shlq $32, %rdx
905 905 orq %rdx, %rax
906 906 ret
907 907 SET_SIZE(randtick)
908 908 #else
909 909 ENTRY_NP(randtick)
910 910 rdtsc
911 911 ret
912 912 SET_SIZE(randtick)
913 913 #endif /* __i386 */
914 914 #endif /* __lint */
915 915 /*
916 916 * Insert entryp after predp in a doubly linked list.
917 917 */
918 918
919 919 #if defined(__lint)
920 920
921 921 /*ARGSUSED*/
922 922 void
923 923 _insque(caddr_t entryp, caddr_t predp)
924 924 {}
925 925
926 926 #else /* __lint */
927 927
928 928 #if defined(__amd64)
929 929
930 930 ENTRY(_insque)
931 931 movq (%rsi), %rax /* predp->forw */
932 932 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
933 933 movq %rax, (%rdi) /* entryp->forw = predp->forw */
934 934 movq %rdi, (%rsi) /* predp->forw = entryp */
935 935 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
936 936 ret
937 937 SET_SIZE(_insque)
938 938
939 939 #elif defined(__i386)
940 940
941 941 ENTRY(_insque)
942 942 movl 8(%esp), %edx
943 943 movl 4(%esp), %ecx
944 944 movl (%edx), %eax /* predp->forw */
945 945 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
946 946 movl %eax, (%ecx) /* entryp->forw = predp->forw */
947 947 movl %ecx, (%edx) /* predp->forw = entryp */
948 948 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
949 949 ret
950 950 SET_SIZE(_insque)
951 951
952 952 #endif /* __i386 */
953 953 #endif /* __lint */
954 954
955 955 /*
956 956 * Remove entryp from a doubly linked list
957 957 */
958 958
959 959 #if defined(__lint)
960 960
961 961 /*ARGSUSED*/
962 962 void
963 963 _remque(caddr_t entryp)
964 964 {}
965 965
966 966 #else /* __lint */
967 967
968 968 #if defined(__amd64)
969 969
970 970 ENTRY(_remque)
971 971 movq (%rdi), %rax /* entry->forw */
972 972 movq CPTRSIZE(%rdi), %rdx /* entry->back */
973 973 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
974 974 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
975 975 ret
976 976 SET_SIZE(_remque)
977 977
978 978 #elif defined(__i386)
979 979
980 980 ENTRY(_remque)
981 981 movl 4(%esp), %ecx
982 982 movl (%ecx), %eax /* entry->forw */
983 983 movl CPTRSIZE(%ecx), %edx /* entry->back */
984 984 movl %eax, (%edx) /* entry->back->forw = entry->forw */
985 985 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
986 986 ret
987 987 SET_SIZE(_remque)
988 988
989 989 #endif /* __i386 */
990 990 #endif /* __lint */
991 991
992 992 /*
993 993 * Returns the number of
994 994 * non-NULL bytes in string argument.
995 995 */
996 996
997 997 #if defined(__lint)
998 998
999 999 /* ARGSUSED */
1000 1000 size_t
1001 1001 strlen(const char *str)
1002 1002 { return (0); }
1003 1003
1004 1004 #else /* __lint */
1005 1005
1006 1006 #if defined(__amd64)
1007 1007
1008 1008 /*
1009 1009 * This is close to a simple transliteration of a C version of this
1010 1010 * routine. We should either just -make- this be a C version, or
1011 1011 * justify having it in assembler by making it significantly faster.
1012 1012 *
1013 1013 * size_t
1014 1014 * strlen(const char *s)
1015 1015 * {
1016 1016 * const char *s0;
1017 1017 * #if defined(DEBUG)
1018 1018 * if ((uintptr_t)s < KERNELBASE)
1019 1019 * panic(.str_panic_msg);
1020 1020 * #endif
1021 1021 * for (s0 = s; *s; s++)
1022 1022 * ;
1023 1023 * return (s - s0);
1024 1024 * }
1025 1025 */
1026 1026
1027 1027 ENTRY(strlen)
1028 1028 #ifdef DEBUG
1029 1029 movq postbootkernelbase(%rip), %rax
1030 1030 cmpq %rax, %rdi
1031 1031 jae str_valid
1032 1032 pushq %rbp
1033 1033 movq %rsp, %rbp
1034 1034 leaq .str_panic_msg(%rip), %rdi
1035 1035 xorl %eax, %eax
1036 1036 call panic
1037 1037 #endif /* DEBUG */
1038 1038 str_valid:
1039 1039 cmpb $0, (%rdi)
1040 1040 movq %rdi, %rax
1041 1041 je .null_found
1042 1042 .align 4
1043 1043 .strlen_loop:
1044 1044 incq %rdi
1045 1045 cmpb $0, (%rdi)
1046 1046 jne .strlen_loop
1047 1047 .null_found:
1048 1048 subq %rax, %rdi
1049 1049 movq %rdi, %rax
1050 1050 ret
1051 1051 SET_SIZE(strlen)
1052 1052
1053 1053 #elif defined(__i386)
1054 1054
1055 1055 ENTRY(strlen)
1056 1056 #ifdef DEBUG
1057 1057 movl postbootkernelbase, %eax
1058 1058 cmpl %eax, 4(%esp)
1059 1059 jae str_valid
1060 1060 pushl %ebp
1061 1061 movl %esp, %ebp
1062 1062 pushl $.str_panic_msg
1063 1063 call panic
1064 1064 #endif /* DEBUG */
1065 1065
1066 1066 str_valid:
1067 1067 movl 4(%esp), %eax /* %eax = string address */
1068 1068 testl $3, %eax /* if %eax not word aligned */
1069 1069 jnz .not_word_aligned /* goto .not_word_aligned */
1070 1070 .align 4
1071 1071 .word_aligned:
1072 1072 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1073 1073 movl $0x7f7f7f7f, %ecx
1074 1074 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1075 1075 addl $4, %eax /* next word */
1076 1076 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1077 1077 orl %edx, %ecx /* %ecx |= %edx */
1078 1078 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1079 1079 cmpl $0x80808080, %ecx /* if no null byte in this word */
1080 1080 je .word_aligned /* goto .word_aligned */
1081 1081 subl $4, %eax /* post-incremented */
1082 1082 .not_word_aligned:
1083 1083 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1084 1084 je .null_found /* goto .null_found */
1085 1085 incl %eax /* next byte */
1086 1086 testl $3, %eax /* if %eax not word aligned */
1087 1087 jnz .not_word_aligned /* goto .not_word_aligned */
1088 1088 jmp .word_aligned /* goto .word_aligned */
1089 1089 .align 4
1090 1090 .null_found:
1091 1091 subl 4(%esp), %eax /* %eax -= string address */
1092 1092 ret
1093 1093 SET_SIZE(strlen)
1094 1094
1095 1095 #endif /* __i386 */
1096 1096
1097 1097 #ifdef DEBUG
1098 1098 .text
1099 1099 .str_panic_msg:
1100 1100 .string "strlen: argument below kernelbase"
1101 1101 #endif /* DEBUG */
1102 1102
1103 1103 #endif /* __lint */
1104 1104
1105 1105 /*
1106 1106 * Berkeley 4.3 introduced symbolically named interrupt levels
1107 1107 * as a way deal with priority in a machine independent fashion.
1108 1108 * Numbered priorities are machine specific, and should be
1109 1109 * discouraged where possible.
1110 1110 *
1111 1111 * Note, for the machine specific priorities there are
1112 1112 * examples listed for devices that use a particular priority.
1113 1113 * It should not be construed that all devices of that
1114 1114 * type should be at that priority. It is currently were
1115 1115 * the current devices fit into the priority scheme based
1116 1116 * upon time criticalness.
1117 1117 *
1118 1118 * The underlying assumption of these assignments is that
1119 1119 * IPL 10 is the highest level from which a device
1120 1120 * routine can call wakeup. Devices that interrupt from higher
1121 1121 * levels are restricted in what they can do. If they need
1122 1122 * kernels services they should schedule a routine at a lower
1123 1123 * level (via software interrupt) to do the required
1124 1124 * processing.
1125 1125 *
1126 1126 * Examples of this higher usage:
1127 1127 * Level Usage
1128 1128 * 14 Profiling clock (and PROM uart polling clock)
1129 1129 * 12 Serial ports
1130 1130 *
1131 1131 * The serial ports request lower level processing on level 6.
1132 1132 *
1133 1133 * Also, almost all splN routines (where N is a number or a
1134 1134 * mnemonic) will do a RAISE(), on the assumption that they are
1135 1135 * never used to lower our priority.
1136 1136 * The exceptions are:
1137 1137 * spl8() Because you can't be above 15 to begin with!
1138 1138 * splzs() Because this is used at boot time to lower our
1139 1139 * priority, to allow the PROM to poll the uart.
1140 1140 * spl0() Used to lower priority to 0.
1141 1141 */
1142 1142
1143 1143 #if defined(__lint)
1144 1144
1145 1145 int spl0(void) { return (0); }
1146 1146 int spl6(void) { return (0); }
1147 1147 int spl7(void) { return (0); }
1148 1148 int spl8(void) { return (0); }
1149 1149 int splhigh(void) { return (0); }
1150 1150 int splhi(void) { return (0); }
1151 1151 int splzs(void) { return (0); }
1152 1152
1153 1153 /* ARGSUSED */
1154 1154 void
1155 1155 splx(int level)
1156 1156 {}
1157 1157
1158 1158 #else /* __lint */
1159 1159
1160 1160 #if defined(__amd64)
1161 1161
1162 1162 #define SETPRI(level) \
1163 1163 movl $/**/level, %edi; /* new priority */ \
1164 1164 jmp do_splx /* redirect to do_splx */
1165 1165
1166 1166 #define RAISE(level) \
1167 1167 movl $/**/level, %edi; /* new priority */ \
1168 1168 jmp splr /* redirect to splr */
1169 1169
1170 1170 #elif defined(__i386)
1171 1171
1172 1172 #define SETPRI(level) \
1173 1173 pushl $/**/level; /* new priority */ \
1174 1174 call do_splx; /* invoke common splx code */ \
1175 1175 addl $4, %esp; /* unstack arg */ \
1176 1176 ret
1177 1177
1178 1178 #define RAISE(level) \
1179 1179 pushl $/**/level; /* new priority */ \
1180 1180 call splr; /* invoke common splr code */ \
1181 1181 addl $4, %esp; /* unstack args */ \
1182 1182 ret
1183 1183
1184 1184 #endif /* __i386 */
1185 1185
1186 1186 /* locks out all interrupts, including memory errors */
1187 1187 ENTRY(spl8)
1188 1188 SETPRI(15)
1189 1189 SET_SIZE(spl8)
1190 1190
1191 1191 /* just below the level that profiling runs */
1192 1192 ENTRY(spl7)
1193 1193 RAISE(13)
1194 1194 SET_SIZE(spl7)
1195 1195
1196 1196 /* sun specific - highest priority onboard serial i/o asy ports */
1197 1197 ENTRY(splzs)
1198 1198 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1199 1199 SET_SIZE(splzs)
1200 1200
1201 1201 ENTRY(splhi)
1202 1202 ALTENTRY(splhigh)
1203 1203 ALTENTRY(spl6)
1204 1204 ALTENTRY(i_ddi_splhigh)
1205 1205
1206 1206 RAISE(DISP_LEVEL)
1207 1207
1208 1208 SET_SIZE(i_ddi_splhigh)
1209 1209 SET_SIZE(spl6)
1210 1210 SET_SIZE(splhigh)
1211 1211 SET_SIZE(splhi)
1212 1212
1213 1213 /* allow all interrupts */
1214 1214 ENTRY(spl0)
1215 1215 SETPRI(0)
1216 1216 SET_SIZE(spl0)
1217 1217
1218 1218
1219 1219 /* splx implementation */
1220 1220 ENTRY(splx)
1221 1221 jmp do_splx /* redirect to common splx code */
1222 1222 SET_SIZE(splx)
1223 1223
1224 1224 #endif /* __lint */
1225 1225
1226 1226 #if defined(__i386)
1227 1227
1228 1228 /*
1229 1229 * Read and write the %gs register
1230 1230 */
1231 1231
1232 1232 #if defined(__lint)
1233 1233
1234 1234 /*ARGSUSED*/
1235 1235 uint16_t
1236 1236 getgs(void)
1237 1237 { return (0); }
1238 1238
1239 1239 /*ARGSUSED*/
1240 1240 void
1241 1241 setgs(uint16_t sel)
1242 1242 {}
1243 1243
1244 1244 #else /* __lint */
1245 1245
1246 1246 ENTRY(getgs)
1247 1247 clr %eax
1248 1248 movw %gs, %ax
1249 1249 ret
1250 1250 SET_SIZE(getgs)
1251 1251
1252 1252 ENTRY(setgs)
1253 1253 movw 4(%esp), %gs
1254 1254 ret
1255 1255 SET_SIZE(setgs)
1256 1256
1257 1257 #endif /* __lint */
1258 1258 #endif /* __i386 */
1259 1259
1260 1260 #if defined(__lint)
1261 1261
1262 1262 void
1263 1263 pc_reset(void)
1264 1264 {}
1265 1265
1266 1266 void
1267 1267 efi_reset(void)
1268 1268 {}
1269 1269
1270 1270 #else /* __lint */
1271 1271
1272 1272 ENTRY(wait_500ms)
1273 1273 #if defined(__amd64)
1274 1274 pushq %rbx
1275 1275 #elif defined(__i386)
1276 1276 push %ebx
1277 1277 #endif
|
↓ open down ↓ |
509 lines elided |
↑ open up ↑ |
1278 1278 movl $50000, %ebx
1279 1279 1:
1280 1280 call tenmicrosec
1281 1281 decl %ebx
1282 1282 jnz 1b
1283 1283 #if defined(__amd64)
1284 1284 popq %rbx
1285 1285 #elif defined(__i386)
1286 1286 pop %ebx
1287 1287 #endif
1288 - ret
1288 + ret
1289 1289 SET_SIZE(wait_500ms)
1290 1290
1291 1291 #define RESET_METHOD_KBC 1
1292 1292 #define RESET_METHOD_PORT92 2
1293 1293 #define RESET_METHOD_PCI 4
1294 1294
1295 1295 DGDEF3(pc_reset_methods, 4, 8)
1296 1296 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1297 1297
1298 1298 ENTRY(pc_reset)
1299 1299
1300 1300 #if defined(__i386)
1301 1301 testl $RESET_METHOD_KBC, pc_reset_methods
1302 1302 #elif defined(__amd64)
1303 1303 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1304 1304 #endif
1305 1305 jz 1f
1306 1306
1307 1307 /
1308 1308 / Try the classic keyboard controller-triggered reset.
1309 1309 /
1310 1310 movw $0x64, %dx
1311 1311 movb $0xfe, %al
1312 1312 outb (%dx)
1313 1313
1314 1314 / Wait up to 500 milliseconds here for the keyboard controller
1315 1315 / to pull the reset line. On some systems where the keyboard
1316 1316 / controller is slow to pull the reset line, the next reset method
1317 1317 / may be executed (which may be bad if those systems hang when the
1318 1318 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1319 1319 / and Ferrari 4000 (doesn't like the cf9 reset method))
1320 1320
1321 1321 call wait_500ms
1322 1322
1323 1323 1:
1324 1324 #if defined(__i386)
1325 1325 testl $RESET_METHOD_PORT92, pc_reset_methods
1326 1326 #elif defined(__amd64)
1327 1327 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1328 1328 #endif
1329 1329 jz 3f
1330 1330
1331 1331 /
1332 1332 / Try port 0x92 fast reset
1333 1333 /
1334 1334 movw $0x92, %dx
1335 1335 inb (%dx)
1336 1336 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1337 1337 je 1f
1338 1338 testb $1, %al / If bit 0
1339 1339 jz 2f / is clear, jump to perform the reset
1340 1340 andb $0xfe, %al / otherwise,
1341 1341 outb (%dx) / clear bit 0 first, then
1342 1342 2:
1343 1343 orb $1, %al / Set bit 0
1344 1344 outb (%dx) / and reset the system
1345 1345 1:
1346 1346
1347 1347 call wait_500ms
1348 1348
1349 1349 3:
1350 1350 #if defined(__i386)
1351 1351 testl $RESET_METHOD_PCI, pc_reset_methods
1352 1352 #elif defined(__amd64)
1353 1353 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1354 1354 #endif
1355 1355 jz 4f
1356 1356
1357 1357 / Try the PCI (soft) reset vector (should work on all modern systems,
1358 1358 / but has been shown to cause problems on 450NX systems, and some newer
1359 1359 / systems (e.g. ATI IXP400-equipped systems))
1360 1360 / When resetting via this method, 2 writes are required. The first
1361 1361 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1362 1362 / power cycle).
1363 1363 / The reset occurs on the second write, during bit 2's transition from
1364 1364 / 0->1.
1365 1365 movw $0xcf9, %dx
1366 1366 movb $0x2, %al / Reset mode = hard, no power cycle
1367 1367 outb (%dx)
1368 1368 movb $0x6, %al
1369 1369 outb (%dx)
1370 1370
1371 1371 call wait_500ms
1372 1372
1373 1373 4:
1374 1374 /
1375 1375 / port 0xcf9 failed also. Last-ditch effort is to
1376 1376 / triple-fault the CPU.
1377 1377 / Also, use triple fault for EFI firmware
1378 1378 /
1379 1379 ENTRY(efi_reset)
1380 1380 #if defined(__amd64)
1381 1381 pushq $0x0
1382 1382 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1383 1383 lidt (%rsp)
1384 1384 #elif defined(__i386)
1385 1385 pushl $0x0
1386 1386 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1387 1387 lidt (%esp)
1388 1388 #endif
1389 1389 int $0x0 / Trigger interrupt, generate triple-fault
1390 1390
1391 1391 cli
1392 1392 hlt / Wait forever
1393 1393 /*NOTREACHED*/
1394 1394 SET_SIZE(efi_reset)
1395 1395 SET_SIZE(pc_reset)
1396 1396
1397 1397 #endif /* __lint */
1398 1398
1399 1399 /*
1400 1400 * C callable in and out routines
1401 1401 */
1402 1402
1403 1403 #if defined(__lint)
1404 1404
1405 1405 /* ARGSUSED */
1406 1406 void
1407 1407 outl(int port_address, uint32_t val)
1408 1408 {}
1409 1409
1410 1410 #else /* __lint */
1411 1411
1412 1412 #if defined(__amd64)
1413 1413
1414 1414 ENTRY(outl)
1415 1415 movw %di, %dx
1416 1416 movl %esi, %eax
1417 1417 outl (%dx)
1418 1418 ret
1419 1419 SET_SIZE(outl)
1420 1420
1421 1421 #elif defined(__i386)
1422 1422
1423 1423 .set PORT, 4
1424 1424 .set VAL, 8
1425 1425
1426 1426 ENTRY(outl)
1427 1427 movw PORT(%esp), %dx
1428 1428 movl VAL(%esp), %eax
1429 1429 outl (%dx)
1430 1430 ret
1431 1431 SET_SIZE(outl)
1432 1432
1433 1433 #endif /* __i386 */
1434 1434 #endif /* __lint */
1435 1435
1436 1436 #if defined(__lint)
1437 1437
1438 1438 /* ARGSUSED */
1439 1439 void
1440 1440 outw(int port_address, uint16_t val)
1441 1441 {}
1442 1442
1443 1443 #else /* __lint */
1444 1444
1445 1445 #if defined(__amd64)
1446 1446
1447 1447 ENTRY(outw)
1448 1448 movw %di, %dx
1449 1449 movw %si, %ax
1450 1450 D16 outl (%dx) /* XX64 why not outw? */
1451 1451 ret
1452 1452 SET_SIZE(outw)
1453 1453
1454 1454 #elif defined(__i386)
1455 1455
1456 1456 ENTRY(outw)
1457 1457 movw PORT(%esp), %dx
1458 1458 movw VAL(%esp), %ax
1459 1459 D16 outl (%dx)
1460 1460 ret
1461 1461 SET_SIZE(outw)
1462 1462
1463 1463 #endif /* __i386 */
1464 1464 #endif /* __lint */
1465 1465
1466 1466 #if defined(__lint)
1467 1467
1468 1468 /* ARGSUSED */
1469 1469 void
1470 1470 outb(int port_address, uint8_t val)
1471 1471 {}
1472 1472
1473 1473 #else /* __lint */
1474 1474
1475 1475 #if defined(__amd64)
1476 1476
1477 1477 ENTRY(outb)
1478 1478 movw %di, %dx
1479 1479 movb %sil, %al
1480 1480 outb (%dx)
1481 1481 ret
1482 1482 SET_SIZE(outb)
1483 1483
1484 1484 #elif defined(__i386)
1485 1485
1486 1486 ENTRY(outb)
1487 1487 movw PORT(%esp), %dx
1488 1488 movb VAL(%esp), %al
1489 1489 outb (%dx)
1490 1490 ret
1491 1491 SET_SIZE(outb)
1492 1492
1493 1493 #endif /* __i386 */
1494 1494 #endif /* __lint */
1495 1495
1496 1496 #if defined(__lint)
1497 1497
1498 1498 /* ARGSUSED */
1499 1499 uint32_t
1500 1500 inl(int port_address)
1501 1501 { return (0); }
1502 1502
1503 1503 #else /* __lint */
1504 1504
1505 1505 #if defined(__amd64)
1506 1506
1507 1507 ENTRY(inl)
1508 1508 xorl %eax, %eax
1509 1509 movw %di, %dx
1510 1510 inl (%dx)
1511 1511 ret
1512 1512 SET_SIZE(inl)
1513 1513
1514 1514 #elif defined(__i386)
1515 1515
1516 1516 ENTRY(inl)
1517 1517 movw PORT(%esp), %dx
1518 1518 inl (%dx)
1519 1519 ret
1520 1520 SET_SIZE(inl)
1521 1521
1522 1522 #endif /* __i386 */
1523 1523 #endif /* __lint */
1524 1524
1525 1525 #if defined(__lint)
1526 1526
1527 1527 /* ARGSUSED */
1528 1528 uint16_t
1529 1529 inw(int port_address)
1530 1530 { return (0); }
1531 1531
1532 1532 #else /* __lint */
1533 1533
1534 1534 #if defined(__amd64)
1535 1535
1536 1536 ENTRY(inw)
1537 1537 xorl %eax, %eax
1538 1538 movw %di, %dx
1539 1539 D16 inl (%dx)
1540 1540 ret
1541 1541 SET_SIZE(inw)
1542 1542
1543 1543 #elif defined(__i386)
1544 1544
1545 1545 ENTRY(inw)
1546 1546 subl %eax, %eax
1547 1547 movw PORT(%esp), %dx
1548 1548 D16 inl (%dx)
1549 1549 ret
1550 1550 SET_SIZE(inw)
1551 1551
1552 1552 #endif /* __i386 */
1553 1553 #endif /* __lint */
1554 1554
1555 1555
1556 1556 #if defined(__lint)
1557 1557
1558 1558 /* ARGSUSED */
1559 1559 uint8_t
1560 1560 inb(int port_address)
1561 1561 { return (0); }
1562 1562
1563 1563 #else /* __lint */
1564 1564
1565 1565 #if defined(__amd64)
1566 1566
1567 1567 ENTRY(inb)
1568 1568 xorl %eax, %eax
1569 1569 movw %di, %dx
1570 1570 inb (%dx)
1571 1571 ret
1572 1572 SET_SIZE(inb)
1573 1573
1574 1574 #elif defined(__i386)
1575 1575
1576 1576 ENTRY(inb)
1577 1577 subl %eax, %eax
1578 1578 movw PORT(%esp), %dx
1579 1579 inb (%dx)
1580 1580 ret
1581 1581 SET_SIZE(inb)
1582 1582
1583 1583 #endif /* __i386 */
1584 1584 #endif /* __lint */
1585 1585
1586 1586
1587 1587 #if defined(__lint)
1588 1588
1589 1589 /* ARGSUSED */
1590 1590 void
1591 1591 repoutsw(int port, uint16_t *addr, int cnt)
1592 1592 {}
1593 1593
1594 1594 #else /* __lint */
1595 1595
1596 1596 #if defined(__amd64)
1597 1597
1598 1598 ENTRY(repoutsw)
1599 1599 movl %edx, %ecx
1600 1600 movw %di, %dx
1601 1601 rep
1602 1602 D16 outsl
1603 1603 ret
1604 1604 SET_SIZE(repoutsw)
1605 1605
1606 1606 #elif defined(__i386)
1607 1607
1608 1608 /*
1609 1609 * The arguments and saved registers are on the stack in the
1610 1610 * following order:
1611 1611 * | cnt | +16
1612 1612 * | *addr | +12
1613 1613 * | port | +8
1614 1614 * | eip | +4
1615 1615 * | esi | <-- %esp
1616 1616 * If additional values are pushed onto the stack, make sure
1617 1617 * to adjust the following constants accordingly.
1618 1618 */
1619 1619 .set PORT, 8
1620 1620 .set ADDR, 12
1621 1621 .set COUNT, 16
1622 1622
1623 1623 ENTRY(repoutsw)
1624 1624 pushl %esi
1625 1625 movl PORT(%esp), %edx
1626 1626 movl ADDR(%esp), %esi
1627 1627 movl COUNT(%esp), %ecx
1628 1628 rep
1629 1629 D16 outsl
1630 1630 popl %esi
1631 1631 ret
1632 1632 SET_SIZE(repoutsw)
1633 1633
1634 1634 #endif /* __i386 */
1635 1635 #endif /* __lint */
1636 1636
1637 1637
1638 1638 #if defined(__lint)
1639 1639
1640 1640 /* ARGSUSED */
1641 1641 void
1642 1642 repinsw(int port_addr, uint16_t *addr, int cnt)
1643 1643 {}
1644 1644
1645 1645 #else /* __lint */
1646 1646
1647 1647 #if defined(__amd64)
1648 1648
1649 1649 ENTRY(repinsw)
1650 1650 movl %edx, %ecx
1651 1651 movw %di, %dx
1652 1652 rep
1653 1653 D16 insl
1654 1654 ret
1655 1655 SET_SIZE(repinsw)
1656 1656
1657 1657 #elif defined(__i386)
1658 1658
1659 1659 ENTRY(repinsw)
1660 1660 pushl %edi
1661 1661 movl PORT(%esp), %edx
1662 1662 movl ADDR(%esp), %edi
1663 1663 movl COUNT(%esp), %ecx
1664 1664 rep
1665 1665 D16 insl
1666 1666 popl %edi
1667 1667 ret
1668 1668 SET_SIZE(repinsw)
1669 1669
1670 1670 #endif /* __i386 */
1671 1671 #endif /* __lint */
1672 1672
1673 1673
1674 1674 #if defined(__lint)
1675 1675
|
↓ open down ↓ |
377 lines elided |
↑ open up ↑ |
1676 1676 /* ARGSUSED */
1677 1677 void
1678 1678 repinsb(int port, uint8_t *addr, int count)
1679 1679 {}
1680 1680
1681 1681 #else /* __lint */
1682 1682
1683 1683 #if defined(__amd64)
1684 1684
1685 1685 ENTRY(repinsb)
1686 - movl %edx, %ecx
1686 + movl %edx, %ecx
1687 1687 movw %di, %dx
1688 1688 movq %rsi, %rdi
1689 1689 rep
1690 1690 insb
1691 - ret
1691 + ret
1692 1692 SET_SIZE(repinsb)
1693 1693
1694 1694 #elif defined(__i386)
1695 -
1695 +
1696 1696 /*
1697 1697 * The arguments and saved registers are on the stack in the
1698 1698 * following order:
1699 1699 * | cnt | +16
1700 1700 * | *addr | +12
1701 1701 * | port | +8
1702 1702 * | eip | +4
1703 1703 * | esi | <-- %esp
1704 1704 * If additional values are pushed onto the stack, make sure
1705 1705 * to adjust the following constants accordingly.
1706 1706 */
1707 1707 .set IO_PORT, 8
1708 1708 .set IO_ADDR, 12
1709 1709 .set IO_COUNT, 16
1710 1710
1711 1711 ENTRY(repinsb)
1712 1712 pushl %edi
1713 1713 movl IO_ADDR(%esp), %edi
1714 1714 movl IO_COUNT(%esp), %ecx
1715 1715 movl IO_PORT(%esp), %edx
1716 1716 rep
1717 1717 insb
1718 1718 popl %edi
1719 1719 ret
1720 1720 SET_SIZE(repinsb)
1721 1721
1722 1722 #endif /* __i386 */
1723 1723 #endif /* __lint */
1724 1724
1725 1725
1726 1726 /*
1727 1727 * Input a stream of 32-bit words.
1728 1728 * NOTE: count is a DWORD count.
1729 1729 */
|
↓ open down ↓ |
24 lines elided |
↑ open up ↑ |
1730 1730 #if defined(__lint)
1731 1731
1732 1732 /* ARGSUSED */
1733 1733 void
1734 1734 repinsd(int port, uint32_t *addr, int count)
1735 1735 {}
1736 1736
1737 1737 #else /* __lint */
1738 1738
1739 1739 #if defined(__amd64)
1740 -
1740 +
1741 1741 ENTRY(repinsd)
1742 1742 movl %edx, %ecx
1743 1743 movw %di, %dx
1744 1744 movq %rsi, %rdi
1745 1745 rep
1746 1746 insl
1747 1747 ret
1748 1748 SET_SIZE(repinsd)
1749 1749
1750 1750 #elif defined(__i386)
1751 1751
1752 1752 ENTRY(repinsd)
1753 1753 pushl %edi
1754 1754 movl IO_ADDR(%esp), %edi
1755 1755 movl IO_COUNT(%esp), %ecx
1756 1756 movl IO_PORT(%esp), %edx
1757 1757 rep
1758 1758 insl
1759 1759 popl %edi
1760 1760 ret
1761 1761 SET_SIZE(repinsd)
1762 1762
1763 1763 #endif /* __i386 */
1764 1764 #endif /* __lint */
1765 1765
1766 1766 /*
1767 1767 * Output a stream of bytes
1768 1768 * NOTE: count is a byte count
1769 1769 */
1770 1770 #if defined(__lint)
1771 1771
1772 1772 /* ARGSUSED */
1773 1773 void
1774 1774 repoutsb(int port, uint8_t *addr, int count)
1775 1775 {}
|
↓ open down ↓ |
25 lines elided |
↑ open up ↑ |
1776 1776
1777 1777 #else /* __lint */
1778 1778
1779 1779 #if defined(__amd64)
1780 1780
1781 1781 ENTRY(repoutsb)
1782 1782 movl %edx, %ecx
1783 1783 movw %di, %dx
1784 1784 rep
1785 1785 outsb
1786 - ret
1786 + ret
1787 1787 SET_SIZE(repoutsb)
1788 1788
1789 1789 #elif defined(__i386)
1790 1790
1791 1791 ENTRY(repoutsb)
1792 1792 pushl %esi
1793 1793 movl IO_ADDR(%esp), %esi
1794 1794 movl IO_COUNT(%esp), %ecx
1795 1795 movl IO_PORT(%esp), %edx
1796 1796 rep
1797 1797 outsb
1798 1798 popl %esi
1799 1799 ret
1800 1800 SET_SIZE(repoutsb)
1801 1801
1802 -#endif /* __i386 */
1802 +#endif /* __i386 */
1803 1803 #endif /* __lint */
1804 1804
1805 1805 /*
1806 1806 * Output a stream of 32-bit words
1807 1807 * NOTE: count is a DWORD count
1808 1808 */
1809 1809 #if defined(__lint)
1810 1810
1811 1811 /* ARGSUSED */
1812 1812 void
1813 1813 repoutsd(int port, uint32_t *addr, int count)
1814 1814 {}
|
↓ open down ↓ |
2 lines elided |
↑ open up ↑ |
1815 1815
1816 1816 #else /* __lint */
1817 1817
1818 1818 #if defined(__amd64)
1819 1819
1820 1820 ENTRY(repoutsd)
1821 1821 movl %edx, %ecx
1822 1822 movw %di, %dx
1823 1823 rep
1824 1824 outsl
1825 - ret
1825 + ret
1826 1826 SET_SIZE(repoutsd)
1827 1827
1828 1828 #elif defined(__i386)
1829 1829
1830 1830 ENTRY(repoutsd)
1831 1831 pushl %esi
1832 1832 movl IO_ADDR(%esp), %esi
1833 1833 movl IO_COUNT(%esp), %ecx
1834 1834 movl IO_PORT(%esp), %edx
1835 1835 rep
1836 1836 outsl
1837 1837 popl %esi
1838 1838 ret
1839 1839 SET_SIZE(repoutsd)
1840 1840
1841 1841 #endif /* __i386 */
1842 1842 #endif /* __lint */
1843 1843
1844 1844 /*
1845 1845 * void int3(void)
1846 1846 * void int18(void)
1847 1847 * void int20(void)
1848 1848 * void int_cmci(void)
1849 1849 */
1850 1850
1851 1851 #if defined(__lint)
1852 1852
1853 1853 void
1854 1854 int3(void)
1855 1855 {}
1856 1856
1857 1857 void
1858 1858 int18(void)
1859 1859 {}
1860 1860
1861 1861 void
1862 1862 int20(void)
1863 1863 {}
1864 1864
1865 1865 void
1866 1866 int_cmci(void)
1867 1867 {}
1868 1868
1869 1869 #else /* __lint */
1870 1870
1871 1871 ENTRY(int3)
1872 1872 int $T_BPTFLT
1873 1873 ret
1874 1874 SET_SIZE(int3)
1875 1875
1876 1876 ENTRY(int18)
1877 1877 int $T_MCE
1878 1878 ret
1879 1879 SET_SIZE(int18)
1880 1880
1881 1881 ENTRY(int20)
1882 1882 movl boothowto, %eax
1883 1883 andl $RB_DEBUG, %eax
1884 1884 jz 1f
1885 1885
1886 1886 int $T_DBGENTR
1887 1887 1:
1888 1888 rep; ret /* use 2 byte return instruction when branch target */
1889 1889 /* AMD Software Optimization Guide - Section 6.2 */
1890 1890 SET_SIZE(int20)
1891 1891
1892 1892 ENTRY(int_cmci)
1893 1893 int $T_ENOEXTFLT
1894 1894 ret
1895 1895 SET_SIZE(int_cmci)
1896 1896
1897 1897 #endif /* __lint */
1898 1898
1899 1899 #if defined(__lint)
1900 1900
1901 1901 /* ARGSUSED */
1902 1902 int
1903 1903 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1904 1904 { return (0); }
1905 1905
1906 1906 #else /* __lint */
1907 1907
1908 1908 #if defined(__amd64)
1909 1909
1910 1910 ENTRY(scanc)
1911 1911 /* rdi == size */
1912 1912 /* rsi == cp */
1913 1913 /* rdx == table */
1914 1914 /* rcx == mask */
1915 1915 addq %rsi, %rdi /* end = &cp[size] */
1916 1916 .scanloop:
1917 1917 cmpq %rdi, %rsi /* while (cp < end */
1918 1918 jnb .scandone
1919 1919 movzbq (%rsi), %r8 /* %r8 = *cp */
1920 1920 incq %rsi /* cp++ */
1921 1921 testb %cl, (%r8, %rdx)
1922 1922 jz .scanloop /* && (table[*cp] & mask) == 0) */
1923 1923 decq %rsi /* (fix post-increment) */
1924 1924 .scandone:
1925 1925 movl %edi, %eax
1926 1926 subl %esi, %eax /* return (end - cp) */
1927 1927 ret
1928 1928 SET_SIZE(scanc)
1929 1929
1930 1930 #elif defined(__i386)
1931 1931
1932 1932 ENTRY(scanc)
1933 1933 pushl %edi
1934 1934 pushl %esi
1935 1935 movb 24(%esp), %cl /* mask = %cl */
1936 1936 movl 16(%esp), %esi /* cp = %esi */
1937 1937 movl 20(%esp), %edx /* table = %edx */
1938 1938 movl %esi, %edi
1939 1939 addl 12(%esp), %edi /* end = &cp[size]; */
1940 1940 .scanloop:
1941 1941 cmpl %edi, %esi /* while (cp < end */
1942 1942 jnb .scandone
1943 1943 movzbl (%esi), %eax /* %al = *cp */
1944 1944 incl %esi /* cp++ */
1945 1945 movb (%edx, %eax), %al /* %al = table[*cp] */
1946 1946 testb %al, %cl
1947 1947 jz .scanloop /* && (table[*cp] & mask) == 0) */
1948 1948 dec %esi /* post-incremented */
1949 1949 .scandone:
1950 1950 movl %edi, %eax
1951 1951 subl %esi, %eax /* return (end - cp) */
1952 1952 popl %esi
1953 1953 popl %edi
1954 1954 ret
1955 1955 SET_SIZE(scanc)
1956 1956
1957 1957 #endif /* __i386 */
1958 1958 #endif /* __lint */
1959 1959
1960 1960 /*
1961 1961 * Replacement functions for ones that are normally inlined.
1962 1962 * In addition to the copy in i86.il, they are defined here just in case.
1963 1963 */
1964 1964
1965 1965 #if defined(__lint)
1966 1966
1967 1967 ulong_t
1968 1968 intr_clear(void)
1969 1969 { return (0); }
1970 1970
1971 1971 ulong_t
1972 1972 clear_int_flag(void)
1973 1973 { return (0); }
1974 1974
1975 1975 #else /* __lint */
1976 1976
1977 1977 #if defined(__amd64)
1978 1978
1979 1979 ENTRY(intr_clear)
1980 1980 ENTRY(clear_int_flag)
1981 1981 pushfq
1982 1982 popq %rax
1983 1983 #if defined(__xpv)
1984 1984 leaq xpv_panicking, %rdi
1985 1985 movl (%rdi), %edi
1986 1986 cmpl $0, %edi
1987 1987 jne 2f
1988 1988 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1989 1989 /*
1990 1990 * Synthesize the PS_IE bit from the event mask bit
1991 1991 */
1992 1992 andq $_BITNOT(PS_IE), %rax
1993 1993 testb $1, %dl
1994 1994 jnz 1f
1995 1995 orq $PS_IE, %rax
1996 1996 1:
1997 1997 ret
1998 1998 2:
1999 1999 #endif
2000 2000 CLI(%rdi)
2001 2001 ret
2002 2002 SET_SIZE(clear_int_flag)
2003 2003 SET_SIZE(intr_clear)
2004 2004
2005 2005 #elif defined(__i386)
2006 2006
2007 2007 ENTRY(intr_clear)
2008 2008 ENTRY(clear_int_flag)
2009 2009 pushfl
2010 2010 popl %eax
2011 2011 #if defined(__xpv)
2012 2012 leal xpv_panicking, %edx
2013 2013 movl (%edx), %edx
2014 2014 cmpl $0, %edx
2015 2015 jne 2f
2016 2016 CLIRET(%edx, %cl) /* returns event mask in %cl */
2017 2017 /*
2018 2018 * Synthesize the PS_IE bit from the event mask bit
2019 2019 */
2020 2020 andl $_BITNOT(PS_IE), %eax
2021 2021 testb $1, %cl
2022 2022 jnz 1f
2023 2023 orl $PS_IE, %eax
2024 2024 1:
2025 2025 ret
2026 2026 2:
2027 2027 #endif
2028 2028 CLI(%edx)
2029 2029 ret
2030 2030 SET_SIZE(clear_int_flag)
2031 2031 SET_SIZE(intr_clear)
2032 2032
2033 2033 #endif /* __i386 */
2034 2034 #endif /* __lint */
2035 2035
2036 2036 #if defined(__lint)
2037 2037
2038 2038 struct cpu *
2039 2039 curcpup(void)
2040 2040 { return 0; }
2041 2041
2042 2042 #else /* __lint */
2043 2043
2044 2044 #if defined(__amd64)
2045 2045
2046 2046 ENTRY(curcpup)
2047 2047 movq %gs:CPU_SELF, %rax
2048 2048 ret
2049 2049 SET_SIZE(curcpup)
2050 2050
2051 2051 #elif defined(__i386)
2052 2052
2053 2053 ENTRY(curcpup)
2054 2054 movl %gs:CPU_SELF, %eax
2055 2055 ret
2056 2056 SET_SIZE(curcpup)
2057 2057
2058 2058 #endif /* __i386 */
2059 2059 #endif /* __lint */
2060 2060
2061 2061 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2062 2062 * These functions reverse the byte order of the input parameter and returns
2063 2063 * the result. This is to convert the byte order from host byte order
2064 2064 * (little endian) to network byte order (big endian), or vice versa.
2065 2065 */
2066 2066
2067 2067 #if defined(__lint)
2068 2068
2069 2069 uint64_t
2070 2070 htonll(uint64_t i)
2071 2071 { return (i); }
2072 2072
2073 2073 uint64_t
2074 2074 ntohll(uint64_t i)
2075 2075 { return (i); }
2076 2076
2077 2077 uint32_t
2078 2078 htonl(uint32_t i)
2079 2079 { return (i); }
2080 2080
2081 2081 uint32_t
2082 2082 ntohl(uint32_t i)
2083 2083 { return (i); }
2084 2084
2085 2085 uint16_t
2086 2086 htons(uint16_t i)
2087 2087 { return (i); }
2088 2088
2089 2089 uint16_t
2090 2090 ntohs(uint16_t i)
2091 2091 { return (i); }
2092 2092
2093 2093 #else /* __lint */
2094 2094
2095 2095 #if defined(__amd64)
2096 2096
2097 2097 ENTRY(htonll)
2098 2098 ALTENTRY(ntohll)
2099 2099 movq %rdi, %rax
2100 2100 bswapq %rax
2101 2101 ret
2102 2102 SET_SIZE(ntohll)
2103 2103 SET_SIZE(htonll)
2104 2104
2105 2105 /* XX64 there must be shorter sequences for this */
2106 2106 ENTRY(htonl)
2107 2107 ALTENTRY(ntohl)
2108 2108 movl %edi, %eax
2109 2109 bswap %eax
2110 2110 ret
2111 2111 SET_SIZE(ntohl)
2112 2112 SET_SIZE(htonl)
2113 2113
2114 2114 /* XX64 there must be better sequences for this */
2115 2115 ENTRY(htons)
2116 2116 ALTENTRY(ntohs)
2117 2117 movl %edi, %eax
2118 2118 bswap %eax
2119 2119 shrl $16, %eax
2120 2120 ret
2121 2121 SET_SIZE(ntohs)
2122 2122 SET_SIZE(htons)
2123 2123
2124 2124 #elif defined(__i386)
2125 2125
2126 2126 ENTRY(htonll)
2127 2127 ALTENTRY(ntohll)
2128 2128 movl 4(%esp), %edx
2129 2129 movl 8(%esp), %eax
2130 2130 bswap %edx
2131 2131 bswap %eax
2132 2132 ret
2133 2133 SET_SIZE(ntohll)
2134 2134 SET_SIZE(htonll)
2135 2135
2136 2136 ENTRY(htonl)
2137 2137 ALTENTRY(ntohl)
2138 2138 movl 4(%esp), %eax
2139 2139 bswap %eax
2140 2140 ret
2141 2141 SET_SIZE(ntohl)
2142 2142 SET_SIZE(htonl)
2143 2143
2144 2144 ENTRY(htons)
2145 2145 ALTENTRY(ntohs)
2146 2146 movl 4(%esp), %eax
2147 2147 bswap %eax
2148 2148 shrl $16, %eax
2149 2149 ret
2150 2150 SET_SIZE(ntohs)
2151 2151 SET_SIZE(htons)
2152 2152
2153 2153 #endif /* __i386 */
2154 2154 #endif /* __lint */
2155 2155
2156 2156
2157 2157 #if defined(__lint)
2158 2158
2159 2159 /* ARGSUSED */
2160 2160 void
2161 2161 intr_restore(ulong_t i)
2162 2162 { return; }
2163 2163
2164 2164 /* ARGSUSED */
2165 2165 void
2166 2166 restore_int_flag(ulong_t i)
2167 2167 { return; }
2168 2168
2169 2169 #else /* __lint */
2170 2170
2171 2171 #if defined(__amd64)
2172 2172
2173 2173 ENTRY(intr_restore)
2174 2174 ENTRY(restore_int_flag)
2175 2175 testq $PS_IE, %rdi
2176 2176 jz 1f
2177 2177 #if defined(__xpv)
2178 2178 leaq xpv_panicking, %rsi
2179 2179 movl (%rsi), %esi
2180 2180 cmpl $0, %esi
2181 2181 jne 1f
2182 2182 /*
2183 2183 * Since we're -really- running unprivileged, our attempt
2184 2184 * to change the state of the IF bit will be ignored.
2185 2185 * The virtual IF bit is tweaked by CLI and STI.
2186 2186 */
2187 2187 IE_TO_EVENT_MASK(%rsi, %rdi)
2188 2188 #else
2189 2189 sti
2190 2190 #endif
2191 2191 1:
2192 2192 ret
2193 2193 SET_SIZE(restore_int_flag)
2194 2194 SET_SIZE(intr_restore)
2195 2195
2196 2196 #elif defined(__i386)
2197 2197
2198 2198 ENTRY(intr_restore)
2199 2199 ENTRY(restore_int_flag)
2200 2200 testl $PS_IE, 4(%esp)
2201 2201 jz 1f
2202 2202 #if defined(__xpv)
2203 2203 leal xpv_panicking, %edx
2204 2204 movl (%edx), %edx
2205 2205 cmpl $0, %edx
2206 2206 jne 1f
2207 2207 /*
2208 2208 * Since we're -really- running unprivileged, our attempt
2209 2209 * to change the state of the IF bit will be ignored.
2210 2210 * The virtual IF bit is tweaked by CLI and STI.
2211 2211 */
2212 2212 IE_TO_EVENT_MASK(%edx, 4(%esp))
2213 2213 #else
2214 2214 sti
2215 2215 #endif
2216 2216 1:
2217 2217 ret
2218 2218 SET_SIZE(restore_int_flag)
2219 2219 SET_SIZE(intr_restore)
2220 2220
2221 2221 #endif /* __i386 */
2222 2222 #endif /* __lint */
2223 2223
2224 2224 #if defined(__lint)
2225 2225
2226 2226 void
2227 2227 sti(void)
2228 2228 {}
2229 2229
2230 2230 void
2231 2231 cli(void)
2232 2232 {}
2233 2233
2234 2234 #else /* __lint */
2235 2235
2236 2236 ENTRY(sti)
2237 2237 STI
2238 2238 ret
2239 2239 SET_SIZE(sti)
2240 2240
2241 2241 ENTRY(cli)
2242 2242 #if defined(__amd64)
2243 2243 CLI(%rax)
2244 2244 #elif defined(__i386)
2245 2245 CLI(%eax)
2246 2246 #endif /* __i386 */
2247 2247 ret
2248 2248 SET_SIZE(cli)
2249 2249
2250 2250 #endif /* __lint */
2251 2251
2252 2252 #if defined(__lint)
2253 2253
2254 2254 dtrace_icookie_t
2255 2255 dtrace_interrupt_disable(void)
2256 2256 { return (0); }
2257 2257
2258 2258 #else /* __lint */
2259 2259
2260 2260 #if defined(__amd64)
2261 2261
2262 2262 ENTRY(dtrace_interrupt_disable)
2263 2263 pushfq
2264 2264 popq %rax
2265 2265 #if defined(__xpv)
2266 2266 leaq xpv_panicking, %rdi
2267 2267 movl (%rdi), %edi
2268 2268 cmpl $0, %edi
2269 2269 jne .dtrace_interrupt_disable_done
2270 2270 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2271 2271 /*
2272 2272 * Synthesize the PS_IE bit from the event mask bit
2273 2273 */
2274 2274 andq $_BITNOT(PS_IE), %rax
2275 2275 testb $1, %dl
|
↓ open down ↓ |
440 lines elided |
↑ open up ↑ |
2276 2276 jnz .dtrace_interrupt_disable_done
2277 2277 orq $PS_IE, %rax
2278 2278 #else
2279 2279 CLI(%rdx)
2280 2280 #endif
2281 2281 .dtrace_interrupt_disable_done:
2282 2282 ret
2283 2283 SET_SIZE(dtrace_interrupt_disable)
2284 2284
2285 2285 #elif defined(__i386)
2286 -
2286 +
2287 2287 ENTRY(dtrace_interrupt_disable)
2288 2288 pushfl
2289 2289 popl %eax
2290 2290 #if defined(__xpv)
2291 2291 leal xpv_panicking, %edx
2292 2292 movl (%edx), %edx
2293 2293 cmpl $0, %edx
2294 2294 jne .dtrace_interrupt_disable_done
2295 2295 CLIRET(%edx, %cl) /* returns event mask in %cl */
2296 2296 /*
2297 2297 * Synthesize the PS_IE bit from the event mask bit
2298 2298 */
2299 2299 andl $_BITNOT(PS_IE), %eax
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
2300 2300 testb $1, %cl
2301 2301 jnz .dtrace_interrupt_disable_done
2302 2302 orl $PS_IE, %eax
2303 2303 #else
2304 2304 CLI(%edx)
2305 2305 #endif
2306 2306 .dtrace_interrupt_disable_done:
2307 2307 ret
2308 2308 SET_SIZE(dtrace_interrupt_disable)
2309 2309
2310 -#endif /* __i386 */
2310 +#endif /* __i386 */
2311 2311 #endif /* __lint */
2312 2312
2313 2313 #if defined(__lint)
2314 2314
2315 2315 /*ARGSUSED*/
2316 2316 void
2317 2317 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2318 2318 {}
2319 2319
2320 2320 #else /* __lint */
2321 2321
2322 2322 #if defined(__amd64)
2323 2323
2324 2324 ENTRY(dtrace_interrupt_enable)
2325 2325 pushq %rdi
2326 2326 popfq
2327 2327 #if defined(__xpv)
2328 2328 leaq xpv_panicking, %rdx
2329 2329 movl (%rdx), %edx
2330 2330 cmpl $0, %edx
2331 2331 jne .dtrace_interrupt_enable_done
2332 2332 /*
2333 2333 * Since we're -really- running unprivileged, our attempt
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
2334 2334 * to change the state of the IF bit will be ignored. The
2335 2335 * virtual IF bit is tweaked by CLI and STI.
2336 2336 */
2337 2337 IE_TO_EVENT_MASK(%rdx, %rdi)
2338 2338 #endif
2339 2339 .dtrace_interrupt_enable_done:
2340 2340 ret
2341 2341 SET_SIZE(dtrace_interrupt_enable)
2342 2342
2343 2343 #elif defined(__i386)
2344 -
2344 +
2345 2345 ENTRY(dtrace_interrupt_enable)
2346 2346 movl 4(%esp), %eax
2347 2347 pushl %eax
2348 2348 popfl
2349 2349 #if defined(__xpv)
2350 2350 leal xpv_panicking, %edx
2351 2351 movl (%edx), %edx
2352 2352 cmpl $0, %edx
2353 2353 jne .dtrace_interrupt_enable_done
2354 2354 /*
2355 2355 * Since we're -really- running unprivileged, our attempt
2356 2356 * to change the state of the IF bit will be ignored. The
2357 2357 * virtual IF bit is tweaked by CLI and STI.
2358 2358 */
2359 2359 IE_TO_EVENT_MASK(%edx, %eax)
2360 2360 #endif
2361 2361 .dtrace_interrupt_enable_done:
2362 2362 ret
2363 2363 SET_SIZE(dtrace_interrupt_enable)
2364 2364
2365 -#endif /* __i386 */
2365 +#endif /* __i386 */
2366 2366 #endif /* __lint */
2367 2367
2368 2368
2369 2369 #if defined(lint)
2370 2370
2371 2371 void
2372 2372 dtrace_membar_producer(void)
2373 2373 {}
2374 2374
2375 2375 void
2376 2376 dtrace_membar_consumer(void)
2377 2377 {}
2378 2378
2379 2379 #else /* __lint */
2380 2380
2381 2381 ENTRY(dtrace_membar_producer)
2382 2382 rep; ret /* use 2 byte return instruction when branch target */
2383 2383 /* AMD Software Optimization Guide - Section 6.2 */
2384 2384 SET_SIZE(dtrace_membar_producer)
2385 2385
2386 2386 ENTRY(dtrace_membar_consumer)
2387 2387 rep; ret /* use 2 byte return instruction when branch target */
2388 2388 /* AMD Software Optimization Guide - Section 6.2 */
2389 2389 SET_SIZE(dtrace_membar_consumer)
2390 2390
2391 2391 #endif /* __lint */
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
2392 2392
2393 2393 #if defined(__lint)
2394 2394
2395 2395 kthread_id_t
2396 2396 threadp(void)
2397 2397 { return ((kthread_id_t)0); }
2398 2398
2399 2399 #else /* __lint */
2400 2400
2401 2401 #if defined(__amd64)
2402 -
2402 +
2403 2403 ENTRY(threadp)
2404 2404 movq %gs:CPU_THREAD, %rax
2405 2405 ret
2406 2406 SET_SIZE(threadp)
2407 2407
2408 2408 #elif defined(__i386)
2409 2409
2410 2410 ENTRY(threadp)
2411 2411 movl %gs:CPU_THREAD, %eax
2412 2412 ret
2413 2413 SET_SIZE(threadp)
2414 2414
2415 2415 #endif /* __i386 */
2416 2416 #endif /* __lint */
2417 2417
2418 2418 /*
2419 2419 * Checksum routine for Internet Protocol Headers
|
↓ open down ↓ |
7 lines elided |
↑ open up ↑ |
2420 2420 */
2421 2421
2422 2422 #if defined(__lint)
2423 2423
2424 2424 /* ARGSUSED */
2425 2425 unsigned int
2426 2426 ip_ocsum(
2427 2427 ushort_t *address, /* ptr to 1st message buffer */
2428 2428 int halfword_count, /* length of data */
2429 2429 unsigned int sum) /* partial checksum */
2430 -{
2430 +{
2431 2431 int i;
2432 2432 unsigned int psum = 0; /* partial sum */
2433 2433
2434 2434 for (i = 0; i < halfword_count; i++, address++) {
2435 2435 psum += *address;
2436 2436 }
2437 2437
2438 2438 while ((psum >> 16) != 0) {
2439 2439 psum = (psum & 0xffff) + (psum >> 16);
2440 2440 }
2441 2441
2442 2442 psum += sum;
2443 2443
2444 2444 while ((psum >> 16) != 0) {
2445 2445 psum = (psum & 0xffff) + (psum >> 16);
2446 2446 }
2447 2447
2448 2448 return (psum);
2449 2449 }
2450 2450
2451 2451 #else /* __lint */
2452 2452
2453 2453 #if defined(__amd64)
2454 2454
2455 2455 ENTRY(ip_ocsum)
2456 2456 pushq %rbp
2457 2457 movq %rsp, %rbp
2458 2458 #ifdef DEBUG
2459 2459 movq postbootkernelbase(%rip), %rax
2460 2460 cmpq %rax, %rdi
2461 2461 jnb 1f
2462 2462 xorl %eax, %eax
2463 2463 movq %rdi, %rsi
2464 2464 leaq .ip_ocsum_panic_msg(%rip), %rdi
2465 2465 call panic
2466 2466 /*NOTREACHED*/
2467 2467 .ip_ocsum_panic_msg:
2468 2468 .string "ip_ocsum: address 0x%p below kernelbase\n"
2469 2469 1:
2470 2470 #endif
2471 2471 movl %esi, %ecx /* halfword_count */
2472 2472 movq %rdi, %rsi /* address */
2473 2473 /* partial sum in %edx */
2474 2474 xorl %eax, %eax
2475 2475 testl %ecx, %ecx
2476 2476 jz .ip_ocsum_done
2477 2477 testq $3, %rsi
2478 2478 jnz .ip_csum_notaligned
2479 2479 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2480 2480 .next_iter:
2481 2481 /* XX64 opportunities for prefetch? */
2482 2482 /* XX64 compute csum with 64 bit quantities? */
2483 2483 subl $32, %ecx
2484 2484 jl .less_than_32
2485 2485
2486 2486 addl 0(%rsi), %edx
2487 2487 .only60:
2488 2488 adcl 4(%rsi), %eax
2489 2489 .only56:
2490 2490 adcl 8(%rsi), %edx
2491 2491 .only52:
2492 2492 adcl 12(%rsi), %eax
2493 2493 .only48:
2494 2494 adcl 16(%rsi), %edx
2495 2495 .only44:
2496 2496 adcl 20(%rsi), %eax
2497 2497 .only40:
2498 2498 adcl 24(%rsi), %edx
2499 2499 .only36:
2500 2500 adcl 28(%rsi), %eax
2501 2501 .only32:
2502 2502 adcl 32(%rsi), %edx
2503 2503 .only28:
2504 2504 adcl 36(%rsi), %eax
2505 2505 .only24:
2506 2506 adcl 40(%rsi), %edx
2507 2507 .only20:
2508 2508 adcl 44(%rsi), %eax
2509 2509 .only16:
2510 2510 adcl 48(%rsi), %edx
2511 2511 .only12:
2512 2512 adcl 52(%rsi), %eax
2513 2513 .only8:
2514 2514 adcl 56(%rsi), %edx
2515 2515 .only4:
2516 2516 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2517 2517 .only0:
2518 2518 adcl $0, %eax /* could be adding -1 in eax with a carry */
2519 2519 adcl $0, %eax
2520 2520
2521 2521 addq $64, %rsi
2522 2522 testl %ecx, %ecx
2523 2523 jnz .next_iter
2524 2524
2525 2525 .ip_ocsum_done:
2526 2526 addl %eax, %edx
2527 2527 adcl $0, %edx
2528 2528 movl %edx, %eax /* form a 16 bit checksum by */
2529 2529 shrl $16, %eax /* adding two halves of 32 bit checksum */
2530 2530 addw %dx, %ax
2531 2531 adcw $0, %ax
2532 2532 andl $0xffff, %eax
2533 2533 leave
2534 2534 ret
2535 2535
2536 2536 .ip_csum_notaligned:
2537 2537 xorl %edi, %edi
2538 2538 movw (%rsi), %di
2539 2539 addl %edi, %edx
2540 2540 adcl $0, %edx
2541 2541 addq $2, %rsi
2542 2542 decl %ecx
2543 2543 jmp .ip_csum_aligned
2544 2544
2545 2545 .less_than_32:
2546 2546 addl $32, %ecx
2547 2547 testl $1, %ecx
2548 2548 jz .size_aligned
2549 2549 andl $0xfe, %ecx
2550 2550 movzwl (%rsi, %rcx, 2), %edi
2551 2551 addl %edi, %edx
2552 2552 adcl $0, %edx
|
↓ open down ↓ |
112 lines elided |
↑ open up ↑ |
2553 2553 .size_aligned:
2554 2554 movl %ecx, %edi
2555 2555 shrl $1, %ecx
2556 2556 shl $1, %edi
2557 2557 subq $64, %rdi
2558 2558 addq %rdi, %rsi
2559 2559 leaq .ip_ocsum_jmptbl(%rip), %rdi
2560 2560 leaq (%rdi, %rcx, 8), %rdi
2561 2561 xorl %ecx, %ecx
2562 2562 clc
2563 - jmp *(%rdi)
2563 + movq (%rdi), %rdi
2564 + INDIRECT_JMP_REG(rdi)
2564 2565
2565 2566 .align 8
2566 2567 .ip_ocsum_jmptbl:
2567 2568 .quad .only0, .only4, .only8, .only12, .only16, .only20
2568 2569 .quad .only24, .only28, .only32, .only36, .only40, .only44
2569 2570 .quad .only48, .only52, .only56, .only60
2570 2571 SET_SIZE(ip_ocsum)
2571 2572
2572 2573 #elif defined(__i386)
2573 2574
2574 2575 ENTRY(ip_ocsum)
2575 2576 pushl %ebp
2576 2577 movl %esp, %ebp
2577 2578 pushl %ebx
2578 2579 pushl %esi
2579 2580 pushl %edi
2580 2581 movl 12(%ebp), %ecx /* count of half words */
2581 2582 movl 16(%ebp), %edx /* partial checksum */
2582 2583 movl 8(%ebp), %esi
2583 2584 xorl %eax, %eax
2584 2585 testl %ecx, %ecx
2585 2586 jz .ip_ocsum_done
2586 2587
2587 2588 testl $3, %esi
2588 2589 jnz .ip_csum_notaligned
2589 2590 .ip_csum_aligned:
2590 2591 .next_iter:
2591 2592 subl $32, %ecx
2592 2593 jl .less_than_32
2593 2594
2594 2595 addl 0(%esi), %edx
2595 2596 .only60:
2596 2597 adcl 4(%esi), %eax
2597 2598 .only56:
2598 2599 adcl 8(%esi), %edx
2599 2600 .only52:
2600 2601 adcl 12(%esi), %eax
2601 2602 .only48:
2602 2603 adcl 16(%esi), %edx
2603 2604 .only44:
2604 2605 adcl 20(%esi), %eax
2605 2606 .only40:
2606 2607 adcl 24(%esi), %edx
2607 2608 .only36:
2608 2609 adcl 28(%esi), %eax
2609 2610 .only32:
2610 2611 adcl 32(%esi), %edx
2611 2612 .only28:
2612 2613 adcl 36(%esi), %eax
2613 2614 .only24:
2614 2615 adcl 40(%esi), %edx
2615 2616 .only20:
2616 2617 adcl 44(%esi), %eax
2617 2618 .only16:
2618 2619 adcl 48(%esi), %edx
2619 2620 .only12:
2620 2621 adcl 52(%esi), %eax
2621 2622 .only8:
2622 2623 adcl 56(%esi), %edx
2623 2624 .only4:
2624 2625 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2625 2626 .only0:
2626 2627 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2627 2628 adcl $0, %eax
2628 2629
2629 2630 addl $64, %esi
2630 2631 andl %ecx, %ecx
2631 2632 jnz .next_iter
2632 2633
2633 2634 .ip_ocsum_done:
2634 2635 addl %eax, %edx
2635 2636 adcl $0, %edx
2636 2637 movl %edx, %eax /* form a 16 bit checksum by */
2637 2638 shrl $16, %eax /* adding two halves of 32 bit checksum */
2638 2639 addw %dx, %ax
2639 2640 adcw $0, %ax
2640 2641 andl $0xffff, %eax
2641 2642 popl %edi /* restore registers */
2642 2643 popl %esi
2643 2644 popl %ebx
2644 2645 leave
2645 2646 ret
2646 2647
2647 2648 .ip_csum_notaligned:
2648 2649 xorl %edi, %edi
2649 2650 movw (%esi), %di
2650 2651 addl %edi, %edx
2651 2652 adcl $0, %edx
2652 2653 addl $2, %esi
2653 2654 decl %ecx
2654 2655 jmp .ip_csum_aligned
2655 2656
2656 2657 .less_than_32:
2657 2658 addl $32, %ecx
2658 2659 testl $1, %ecx
2659 2660 jz .size_aligned
2660 2661 andl $0xfe, %ecx
2661 2662 movzwl (%esi, %ecx, 2), %edi
2662 2663 addl %edi, %edx
2663 2664 adcl $0, %edx
|
↓ open down ↓ |
90 lines elided |
↑ open up ↑ |
2664 2665 .size_aligned:
2665 2666 movl %ecx, %edi
2666 2667 shrl $1, %ecx
2667 2668 shl $1, %edi
2668 2669 subl $64, %edi
2669 2670 addl %edi, %esi
2670 2671 movl $.ip_ocsum_jmptbl, %edi
2671 2672 lea (%edi, %ecx, 4), %edi
2672 2673 xorl %ecx, %ecx
2673 2674 clc
2674 - jmp *(%edi)
2675 + jmp *(%edi)
2675 2676 SET_SIZE(ip_ocsum)
2676 2677
2677 2678 .data
2678 2679 .align 4
2679 2680
2680 2681 .ip_ocsum_jmptbl:
2681 2682 .long .only0, .only4, .only8, .only12, .only16, .only20
2682 2683 .long .only24, .only28, .only32, .only36, .only40, .only44
2683 2684 .long .only48, .only52, .only56, .only60
2684 2685
2685 -
2686 -#endif /* __i386 */
2686 +
2687 +#endif /* __i386 */
2687 2688 #endif /* __lint */
2688 2689
2689 2690 /*
2690 2691 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2691 2692 * Provided to manipulate hrtime_t values.
2692 2693 */
2693 2694 #if defined(__lint)
2694 2695
2695 2696 /* result = a * b; */
2696 2697
2697 2698 /* ARGSUSED */
2698 2699 unsigned long long
2699 2700 mul32(uint_t a, uint_t b)
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
2700 2701 { return (0); }
2701 2702
2702 2703 #else /* __lint */
2703 2704
2704 2705 #if defined(__amd64)
2705 2706
2706 2707 ENTRY(mul32)
2707 2708 xorl %edx, %edx /* XX64 joe, paranoia? */
2708 2709 movl %edi, %eax
2709 2710 mull %esi
2710 - shlq $32, %rdx
2711 + shlq $32, %rdx
2711 2712 orq %rdx, %rax
2712 2713 ret
2713 2714 SET_SIZE(mul32)
2714 2715
2715 2716 #elif defined(__i386)
2716 2717
2717 2718 ENTRY(mul32)
2718 2719 movl 8(%esp), %eax
2719 2720 movl 4(%esp), %ecx
2720 2721 mull %ecx
2721 2722 ret
2722 2723 SET_SIZE(mul32)
2723 2724
2724 2725 #endif /* __i386 */
2725 2726 #endif /* __lint */
2726 2727
2727 2728 #if defined(notused)
2728 2729 #if defined(__lint)
2729 2730 /* ARGSUSED */
2730 2731 void
2731 2732 load_pte64(uint64_t *pte, uint64_t pte_value)
2732 2733 {}
2733 2734 #else /* __lint */
2734 2735 .globl load_pte64
2735 2736 load_pte64:
2736 2737 movl 4(%esp), %eax
2737 2738 movl 8(%esp), %ecx
2738 2739 movl 12(%esp), %edx
2739 2740 movl %edx, 4(%eax)
2740 2741 movl %ecx, (%eax)
2741 2742 ret
2742 2743 #endif /* __lint */
2743 2744 #endif /* notused */
2744 2745
2745 2746 #if defined(__lint)
2746 2747
2747 2748 /*ARGSUSED*/
2748 2749 void
2749 2750 scan_memory(caddr_t addr, size_t size)
2750 2751 {}
2751 2752
2752 2753 #else /* __lint */
2753 2754
2754 2755 #if defined(__amd64)
2755 2756
2756 2757 ENTRY(scan_memory)
2757 2758 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2758 2759 jz .scanm_done
2759 2760 movq %rsi, %rcx /* move count into rep control register */
2760 2761 movq %rdi, %rsi /* move addr into lodsq control reg. */
2761 2762 rep lodsq /* scan the memory range */
2762 2763 .scanm_done:
2763 2764 rep; ret /* use 2 byte return instruction when branch target */
2764 2765 /* AMD Software Optimization Guide - Section 6.2 */
2765 2766 SET_SIZE(scan_memory)
2766 2767
2767 2768 #elif defined(__i386)
2768 2769
2769 2770 ENTRY(scan_memory)
2770 2771 pushl %ecx
2771 2772 pushl %esi
2772 2773 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2773 2774 shrl $2, %ecx /* convert from byte count to word count */
2774 2775 jz .scanm_done
2775 2776 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2776 2777 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2777 2778 lodsl
2778 2779 .scanm_done:
2779 2780 popl %esi
2780 2781 popl %ecx
2781 2782 ret
2782 2783 SET_SIZE(scan_memory)
2783 2784
2784 2785 #endif /* __i386 */
2785 2786 #endif /* __lint */
2786 2787
2787 2788
2788 2789 #if defined(__lint)
2789 2790
2790 2791 /*ARGSUSED */
2791 2792 int
2792 2793 lowbit(ulong_t i)
2793 2794 { return (0); }
2794 2795
2795 2796 #else /* __lint */
2796 2797
2797 2798 #if defined(__amd64)
2798 2799
2799 2800 ENTRY(lowbit)
2800 2801 movl $-1, %eax
2801 2802 bsfq %rdi, %rdi
2802 2803 cmovnz %edi, %eax
2803 2804 incl %eax
2804 2805 ret
2805 2806 SET_SIZE(lowbit)
2806 2807
2807 2808 #elif defined(__i386)
2808 2809
2809 2810 ENTRY(lowbit)
2810 2811 bsfl 4(%esp), %eax
2811 2812 jz 0f
2812 2813 incl %eax
2813 2814 ret
2814 2815 0:
2815 2816 xorl %eax, %eax
2816 2817 ret
2817 2818 SET_SIZE(lowbit)
2818 2819
2819 2820 #endif /* __i386 */
2820 2821 #endif /* __lint */
2821 2822
2822 2823 #if defined(__lint)
2823 2824
2824 2825 /*ARGSUSED*/
2825 2826 int
2826 2827 highbit(ulong_t i)
2827 2828 { return (0); }
2828 2829
2829 2830 /*ARGSUSED*/
2830 2831 int
2831 2832 highbit64(uint64_t i)
2832 2833 { return (0); }
2833 2834
2834 2835 #else /* __lint */
2835 2836
2836 2837 #if defined(__amd64)
2837 2838
2838 2839 ENTRY(highbit)
2839 2840 ALTENTRY(highbit64)
2840 2841 movl $-1, %eax
2841 2842 bsrq %rdi, %rdi
2842 2843 cmovnz %edi, %eax
2843 2844 incl %eax
2844 2845 ret
2845 2846 SET_SIZE(highbit64)
2846 2847 SET_SIZE(highbit)
|
↓ open down ↓ |
126 lines elided |
↑ open up ↑ |
2847 2848
2848 2849 #elif defined(__i386)
2849 2850
2850 2851 ENTRY(highbit)
2851 2852 bsrl 4(%esp), %eax
2852 2853 jz 0f
2853 2854 incl %eax
2854 2855 ret
2855 2856 0:
2856 2857 xorl %eax, %eax
2857 - ret
2858 + ret
2858 2859 SET_SIZE(highbit)
2859 2860
2860 2861 ENTRY(highbit64)
2861 2862 bsrl 8(%esp), %eax
2862 2863 jz highbit
2863 2864 addl $33, %eax
2864 2865 ret
2865 2866 SET_SIZE(highbit64)
2866 2867
2867 2868 #endif /* __i386 */
2868 2869 #endif /* __lint */
2869 2870
2870 2871 #if defined(__lint)
2871 2872
2872 2873 /*ARGSUSED*/
2873 2874 uint64_t
2874 2875 rdmsr(uint_t r)
2875 2876 { return (0); }
2876 2877
2877 2878 /*ARGSUSED*/
2878 2879 void
2879 2880 wrmsr(uint_t r, const uint64_t val)
2880 2881 {}
2881 2882
2882 2883 /*ARGSUSED*/
2883 2884 uint64_t
2884 2885 xrdmsr(uint_t r)
2885 2886 { return (0); }
2886 2887
2887 2888 /*ARGSUSED*/
2888 2889 void
2889 2890 xwrmsr(uint_t r, const uint64_t val)
2890 2891 {}
2891 2892
2892 2893 void
2893 2894 invalidate_cache(void)
2894 2895 {}
2895 2896
2896 2897 /*ARGSUSED*/
2897 2898 uint64_t
2898 2899 get_xcr(uint_t r)
2899 2900 { return (0); }
2900 2901
|
↓ open down ↓ |
33 lines elided |
↑ open up ↑ |
2901 2902 /*ARGSUSED*/
2902 2903 void
2903 2904 set_xcr(uint_t r, const uint64_t val)
2904 2905 {}
2905 2906
2906 2907 #else /* __lint */
2907 2908
2908 2909 #define XMSR_ACCESS_VAL $0x9c5a203a
2909 2910
2910 2911 #if defined(__amd64)
2911 -
2912 +
2912 2913 ENTRY(rdmsr)
2913 2914 movl %edi, %ecx
2914 2915 rdmsr
2915 2916 shlq $32, %rdx
2916 2917 orq %rdx, %rax
2917 2918 ret
2918 2919 SET_SIZE(rdmsr)
2919 2920
2920 2921 ENTRY(wrmsr)
2921 2922 movq %rsi, %rdx
2922 2923 shrq $32, %rdx
2923 2924 movl %esi, %eax
2924 2925 movl %edi, %ecx
2925 2926 wrmsr
2926 2927 ret
2927 2928 SET_SIZE(wrmsr)
2928 2929
2929 2930 ENTRY(xrdmsr)
2930 2931 pushq %rbp
2931 2932 movq %rsp, %rbp
2932 2933 movl %edi, %ecx
2933 2934 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2934 2935 rdmsr
2935 2936 shlq $32, %rdx
2936 2937 orq %rdx, %rax
2937 2938 leave
2938 2939 ret
2939 2940 SET_SIZE(xrdmsr)
2940 2941
2941 2942 ENTRY(xwrmsr)
2942 2943 pushq %rbp
2943 2944 movq %rsp, %rbp
2944 2945 movl %edi, %ecx
2945 2946 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2946 2947 movq %rsi, %rdx
2947 2948 shrq $32, %rdx
2948 2949 movl %esi, %eax
2949 2950 wrmsr
2950 2951 leave
2951 2952 ret
2952 2953 SET_SIZE(xwrmsr)
2953 2954
2954 2955 ENTRY(get_xcr)
2955 2956 movl %edi, %ecx
2956 2957 #xgetbv
2957 2958 .byte 0x0f,0x01,0xd0
2958 2959 shlq $32, %rdx
2959 2960 orq %rdx, %rax
2960 2961 ret
2961 2962 SET_SIZE(get_xcr)
2962 2963
2963 2964 ENTRY(set_xcr)
2964 2965 movq %rsi, %rdx
2965 2966 shrq $32, %rdx
2966 2967 movl %esi, %eax
2967 2968 movl %edi, %ecx
2968 2969 #xsetbv
2969 2970 .byte 0x0f,0x01,0xd1
2970 2971 ret
2971 2972 SET_SIZE(set_xcr)
2972 2973
2973 2974 #elif defined(__i386)
|
↓ open down ↓ |
52 lines elided |
↑ open up ↑ |
2974 2975
2975 2976 ENTRY(rdmsr)
2976 2977 movl 4(%esp), %ecx
2977 2978 rdmsr
2978 2979 ret
2979 2980 SET_SIZE(rdmsr)
2980 2981
2981 2982 ENTRY(wrmsr)
2982 2983 movl 4(%esp), %ecx
2983 2984 movl 8(%esp), %eax
2984 - movl 12(%esp), %edx
2985 + movl 12(%esp), %edx
2985 2986 wrmsr
2986 2987 ret
2987 2988 SET_SIZE(wrmsr)
2988 2989
2989 2990 ENTRY(xrdmsr)
2990 2991 pushl %ebp
2991 2992 movl %esp, %ebp
2992 2993 movl 8(%esp), %ecx
2993 2994 pushl %edi
2994 2995 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2995 2996 rdmsr
|
↓ open down ↓ |
1 lines elided |
↑ open up ↑ |
2996 2997 popl %edi
2997 2998 leave
2998 2999 ret
2999 3000 SET_SIZE(xrdmsr)
3000 3001
3001 3002 ENTRY(xwrmsr)
3002 3003 pushl %ebp
3003 3004 movl %esp, %ebp
3004 3005 movl 8(%esp), %ecx
3005 3006 movl 12(%esp), %eax
3006 - movl 16(%esp), %edx
3007 + movl 16(%esp), %edx
3007 3008 pushl %edi
3008 3009 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3009 3010 wrmsr
3010 3011 popl %edi
3011 3012 leave
3012 3013 ret
3013 3014 SET_SIZE(xwrmsr)
3014 3015
3015 3016 ENTRY(get_xcr)
3016 3017 movl 4(%esp), %ecx
3017 3018 #xgetbv
3018 3019 .byte 0x0f,0x01,0xd0
3019 3020 ret
3020 3021 SET_SIZE(get_xcr)
3021 3022
3022 3023 ENTRY(set_xcr)
3023 3024 movl 4(%esp), %ecx
3024 3025 movl 8(%esp), %eax
3025 3026 movl 12(%esp), %edx
3026 3027 #xsetbv
3027 3028 .byte 0x0f,0x01,0xd1
3028 3029 ret
3029 3030 SET_SIZE(set_xcr)
3030 3031
3031 3032 #endif /* __i386 */
3032 3033
3033 3034 ENTRY(invalidate_cache)
3034 3035 wbinvd
3035 3036 ret
3036 3037 SET_SIZE(invalidate_cache)
3037 3038
3038 3039 #endif /* __lint */
3039 3040
3040 3041 #if defined(__lint)
3041 3042
3042 3043 /*ARGSUSED*/
3043 3044 void
3044 3045 getcregs(struct cregs *crp)
3045 3046 {}
3046 3047
3047 3048 #else /* __lint */
3048 3049
3049 3050 #if defined(__amd64)
3050 3051
3051 3052 ENTRY_NP(getcregs)
3052 3053 #if defined(__xpv)
3053 3054 /*
3054 3055 * Only a few of the hardware control registers or descriptor tables
3055 3056 * are directly accessible to us, so just zero the structure.
3056 3057 *
3057 3058 * XXPV Perhaps it would be helpful for the hypervisor to return
3058 3059 * virtualized versions of these for post-mortem use.
3059 3060 * (Need to reevaluate - perhaps it already does!)
3060 3061 */
3061 3062 pushq %rdi /* save *crp */
3062 3063 movq $CREGSZ, %rsi
3063 3064 call bzero
3064 3065 popq %rdi
3065 3066
3066 3067 /*
3067 3068 * Dump what limited information we can
3068 3069 */
3069 3070 movq %cr0, %rax
3070 3071 movq %rax, CREG_CR0(%rdi) /* cr0 */
3071 3072 movq %cr2, %rax
3072 3073 movq %rax, CREG_CR2(%rdi) /* cr2 */
3073 3074 movq %cr3, %rax
3074 3075 movq %rax, CREG_CR3(%rdi) /* cr3 */
3075 3076 movq %cr4, %rax
3076 3077 movq %rax, CREG_CR4(%rdi) /* cr4 */
3077 3078
3078 3079 #else /* __xpv */
3079 3080
3080 3081 #define GETMSR(r, off, d) \
3081 3082 movl $r, %ecx; \
3082 3083 rdmsr; \
3083 3084 movl %eax, off(d); \
3084 3085 movl %edx, off+4(d)
3085 3086
3086 3087 xorl %eax, %eax
3087 3088 movq %rax, CREG_GDT+8(%rdi)
3088 3089 sgdt CREG_GDT(%rdi) /* 10 bytes */
3089 3090 movq %rax, CREG_IDT+8(%rdi)
3090 3091 sidt CREG_IDT(%rdi) /* 10 bytes */
3091 3092 movq %rax, CREG_LDT(%rdi)
3092 3093 sldt CREG_LDT(%rdi) /* 2 bytes */
3093 3094 movq %rax, CREG_TASKR(%rdi)
3094 3095 str CREG_TASKR(%rdi) /* 2 bytes */
3095 3096 movq %cr0, %rax
3096 3097 movq %rax, CREG_CR0(%rdi) /* cr0 */
3097 3098 movq %cr2, %rax
3098 3099 movq %rax, CREG_CR2(%rdi) /* cr2 */
3099 3100 movq %cr3, %rax
3100 3101 movq %rax, CREG_CR3(%rdi) /* cr3 */
3101 3102 movq %cr4, %rax
3102 3103 movq %rax, CREG_CR4(%rdi) /* cr4 */
3103 3104 movq %cr8, %rax
3104 3105 movq %rax, CREG_CR8(%rdi) /* cr8 */
3105 3106 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3106 3107 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3107 3108 #endif /* __xpv */
3108 3109 ret
3109 3110 SET_SIZE(getcregs)
3110 3111
3111 3112 #undef GETMSR
3112 3113
3113 3114 #elif defined(__i386)
3114 3115
3115 3116 ENTRY_NP(getcregs)
3116 3117 #if defined(__xpv)
3117 3118 /*
3118 3119 * Only a few of the hardware control registers or descriptor tables
3119 3120 * are directly accessible to us, so just zero the structure.
3120 3121 *
3121 3122 * XXPV Perhaps it would be helpful for the hypervisor to return
3122 3123 * virtualized versions of these for post-mortem use.
3123 3124 * (Need to reevaluate - perhaps it already does!)
3124 3125 */
3125 3126 movl 4(%esp), %edx
3126 3127 pushl $CREGSZ
3127 3128 pushl %edx
3128 3129 call bzero
3129 3130 addl $8, %esp
3130 3131 movl 4(%esp), %edx
3131 3132
3132 3133 /*
3133 3134 * Dump what limited information we can
3134 3135 */
3135 3136 movl %cr0, %eax
3136 3137 movl %eax, CREG_CR0(%edx) /* cr0 */
3137 3138 movl %cr2, %eax
3138 3139 movl %eax, CREG_CR2(%edx) /* cr2 */
3139 3140 movl %cr3, %eax
3140 3141 movl %eax, CREG_CR3(%edx) /* cr3 */
3141 3142 movl %cr4, %eax
3142 3143 movl %eax, CREG_CR4(%edx) /* cr4 */
3143 3144
3144 3145 #else /* __xpv */
3145 3146
3146 3147 movl 4(%esp), %edx
3147 3148 movw $0, CREG_GDT+6(%edx)
3148 3149 movw $0, CREG_IDT+6(%edx)
3149 3150 sgdt CREG_GDT(%edx) /* gdt */
3150 3151 sidt CREG_IDT(%edx) /* idt */
3151 3152 sldt CREG_LDT(%edx) /* ldt */
3152 3153 str CREG_TASKR(%edx) /* task */
3153 3154 movl %cr0, %eax
3154 3155 movl %eax, CREG_CR0(%edx) /* cr0 */
3155 3156 movl %cr2, %eax
3156 3157 movl %eax, CREG_CR2(%edx) /* cr2 */
3157 3158 movl %cr3, %eax
3158 3159 movl %eax, CREG_CR3(%edx) /* cr3 */
3159 3160 bt $X86FSET_LARGEPAGE, x86_featureset
3160 3161 jnc .nocr4
3161 3162 movl %cr4, %eax
3162 3163 movl %eax, CREG_CR4(%edx) /* cr4 */
3163 3164 jmp .skip
3164 3165 .nocr4:
3165 3166 movl $0, CREG_CR4(%edx)
3166 3167 .skip:
3167 3168 #endif
3168 3169 ret
3169 3170 SET_SIZE(getcregs)
3170 3171
3171 3172 #endif /* __i386 */
3172 3173 #endif /* __lint */
3173 3174
3174 3175
3175 3176 /*
3176 3177 * A panic trigger is a word which is updated atomically and can only be set
3177 3178 * once. We atomically store 0xDEFACEDD and load the old value. If the
3178 3179 * previous value was 0, we succeed and return 1; otherwise return 0.
3179 3180 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3180 3181 * has its own version of this function to allow it to panic correctly from
3181 3182 * probe context.
3182 3183 */
3183 3184 #if defined(__lint)
3184 3185
3185 3186 /*ARGSUSED*/
3186 3187 int
3187 3188 panic_trigger(int *tp)
3188 3189 { return (0); }
3189 3190
3190 3191 /*ARGSUSED*/
3191 3192 int
3192 3193 dtrace_panic_trigger(int *tp)
3193 3194 { return (0); }
3194 3195
|
↓ open down ↓ |
178 lines elided |
↑ open up ↑ |
3195 3196 #else /* __lint */
3196 3197
3197 3198 #if defined(__amd64)
3198 3199
3199 3200 ENTRY_NP(panic_trigger)
3200 3201 xorl %eax, %eax
3201 3202 movl $0xdefacedd, %edx
3202 3203 lock
3203 3204 xchgl %edx, (%rdi)
3204 3205 cmpl $0, %edx
3205 - je 0f
3206 + je 0f
3206 3207 movl $0, %eax
3207 3208 ret
3208 3209 0: movl $1, %eax
3209 3210 ret
3210 3211 SET_SIZE(panic_trigger)
3211 -
3212 +
3212 3213 ENTRY_NP(dtrace_panic_trigger)
3213 3214 xorl %eax, %eax
3214 3215 movl $0xdefacedd, %edx
3215 3216 lock
3216 3217 xchgl %edx, (%rdi)
3217 3218 cmpl $0, %edx
3218 3219 je 0f
3219 3220 movl $0, %eax
3220 3221 ret
3221 3222 0: movl $1, %eax
3222 3223 ret
3223 3224 SET_SIZE(dtrace_panic_trigger)
3224 3225
3225 3226 #elif defined(__i386)
3226 3227
3227 3228 ENTRY_NP(panic_trigger)
3228 3229 movl 4(%esp), %edx / %edx = address of trigger
3229 3230 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3230 3231 lock / assert lock
3231 3232 xchgl %eax, (%edx) / exchange %eax and the trigger
3232 3233 cmpl $0, %eax / if (%eax == 0x0)
3233 3234 je 0f / return (1);
3234 3235 movl $0, %eax / else
3235 3236 ret / return (0);
3236 3237 0: movl $1, %eax
3237 3238 ret
3238 3239 SET_SIZE(panic_trigger)
3239 3240
3240 3241 ENTRY_NP(dtrace_panic_trigger)
3241 3242 movl 4(%esp), %edx / %edx = address of trigger
3242 3243 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3243 3244 lock / assert lock
3244 3245 xchgl %eax, (%edx) / exchange %eax and the trigger
3245 3246 cmpl $0, %eax / if (%eax == 0x0)
3246 3247 je 0f / return (1);
3247 3248 movl $0, %eax / else
3248 3249 ret / return (0);
3249 3250 0: movl $1, %eax
3250 3251 ret
3251 3252 SET_SIZE(dtrace_panic_trigger)
3252 3253
3253 3254 #endif /* __i386 */
3254 3255 #endif /* __lint */
3255 3256
3256 3257 /*
3257 3258 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3258 3259 * into the panic code implemented in panicsys(). vpanic() is responsible
3259 3260 * for passing through the format string and arguments, and constructing a
3260 3261 * regs structure on the stack into which it saves the current register
3261 3262 * values. If we are not dying due to a fatal trap, these registers will
3262 3263 * then be preserved in panicbuf as the current processor state. Before
3263 3264 * invoking panicsys(), vpanic() activates the first panic trigger (see
3264 3265 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3265 3266 * DTrace takes a slightly different panic path if it must panic from probe
3266 3267 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3267 3268 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3268 3269 * branches back into vpanic().
3269 3270 */
3270 3271 #if defined(__lint)
3271 3272
3272 3273 /*ARGSUSED*/
3273 3274 void
3274 3275 vpanic(const char *format, va_list alist)
3275 3276 {}
3276 3277
|
↓ open down ↓ |
55 lines elided |
↑ open up ↑ |
3277 3278 /*ARGSUSED*/
3278 3279 void
3279 3280 dtrace_vpanic(const char *format, va_list alist)
3280 3281 {}
3281 3282
3282 3283 #else /* __lint */
3283 3284
3284 3285 #if defined(__amd64)
3285 3286
3286 3287 ENTRY_NP(vpanic) /* Initial stack layout: */
3287 -
3288 - pushq %rbp /* | %rip | 0x60 */
3288 +
3289 + pushq %rbp /* | %rip | 0x60 */
3289 3290 movq %rsp, %rbp /* | %rbp | 0x58 */
3290 3291 pushfq /* | rfl | 0x50 */
3291 3292 pushq %r11 /* | %r11 | 0x48 */
3292 3293 pushq %r10 /* | %r10 | 0x40 */
3293 3294 pushq %rbx /* | %rbx | 0x38 */
3294 3295 pushq %rax /* | %rax | 0x30 */
3295 3296 pushq %r9 /* | %r9 | 0x28 */
3296 3297 pushq %r8 /* | %r8 | 0x20 */
3297 3298 pushq %rcx /* | %rcx | 0x18 */
3298 3299 pushq %rdx /* | %rdx | 0x10 */
3299 3300 pushq %rsi /* | %rsi | 0x8 alist */
3300 3301 pushq %rdi /* | %rdi | 0x0 format */
3301 3302
3302 3303 movq %rsp, %rbx /* %rbx = current %rsp */
3303 3304
3304 3305 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3305 3306 call panic_trigger /* %eax = panic_trigger() */
3306 3307
3307 3308 vpanic_common:
3308 3309 /*
3309 3310 * The panic_trigger result is in %eax from the call above, and
3310 3311 * dtrace_panic places it in %eax before branching here.
3311 3312 * The rdmsr instructions that follow below will clobber %eax so
3312 3313 * we stash the panic_trigger result in %r11d.
3313 3314 */
3314 3315 movl %eax, %r11d
3315 3316 cmpl $0, %r11d
3316 3317 je 0f
3317 3318
3318 3319 /*
3319 3320 * If panic_trigger() was successful, we are the first to initiate a
3320 3321 * panic: we now switch to the reserved panic_stack before continuing.
3321 3322 */
3322 3323 leaq panic_stack(%rip), %rsp
3323 3324 addq $PANICSTKSIZE, %rsp
3324 3325 0: subq $REGSIZE, %rsp
3325 3326 /*
3326 3327 * Now that we've got everything set up, store the register values as
3327 3328 * they were when we entered vpanic() to the designated location in
3328 3329 * the regs structure we allocated on the stack.
3329 3330 */
3330 3331 movq 0x0(%rbx), %rcx
3331 3332 movq %rcx, REGOFF_RDI(%rsp)
3332 3333 movq 0x8(%rbx), %rcx
3333 3334 movq %rcx, REGOFF_RSI(%rsp)
3334 3335 movq 0x10(%rbx), %rcx
3335 3336 movq %rcx, REGOFF_RDX(%rsp)
3336 3337 movq 0x18(%rbx), %rcx
3337 3338 movq %rcx, REGOFF_RCX(%rsp)
3338 3339 movq 0x20(%rbx), %rcx
3339 3340
3340 3341 movq %rcx, REGOFF_R8(%rsp)
3341 3342 movq 0x28(%rbx), %rcx
3342 3343 movq %rcx, REGOFF_R9(%rsp)
3343 3344 movq 0x30(%rbx), %rcx
3344 3345 movq %rcx, REGOFF_RAX(%rsp)
3345 3346 movq 0x38(%rbx), %rcx
3346 3347 movq %rcx, REGOFF_RBX(%rsp)
3347 3348 movq 0x58(%rbx), %rcx
3348 3349
3349 3350 movq %rcx, REGOFF_RBP(%rsp)
3350 3351 movq 0x40(%rbx), %rcx
3351 3352 movq %rcx, REGOFF_R10(%rsp)
3352 3353 movq 0x48(%rbx), %rcx
3353 3354 movq %rcx, REGOFF_R11(%rsp)
3354 3355 movq %r12, REGOFF_R12(%rsp)
3355 3356
3356 3357 movq %r13, REGOFF_R13(%rsp)
3357 3358 movq %r14, REGOFF_R14(%rsp)
3358 3359 movq %r15, REGOFF_R15(%rsp)
3359 3360
3360 3361 xorl %ecx, %ecx
3361 3362 movw %ds, %cx
3362 3363 movq %rcx, REGOFF_DS(%rsp)
3363 3364 movw %es, %cx
3364 3365 movq %rcx, REGOFF_ES(%rsp)
3365 3366 movw %fs, %cx
3366 3367 movq %rcx, REGOFF_FS(%rsp)
3367 3368 movw %gs, %cx
3368 3369 movq %rcx, REGOFF_GS(%rsp)
3369 3370
3370 3371 movq $0, REGOFF_TRAPNO(%rsp)
3371 3372
3372 3373 movq $0, REGOFF_ERR(%rsp)
3373 3374 leaq vpanic(%rip), %rcx
3374 3375 movq %rcx, REGOFF_RIP(%rsp)
3375 3376 movw %cs, %cx
3376 3377 movzwq %cx, %rcx
3377 3378 movq %rcx, REGOFF_CS(%rsp)
|
↓ open down ↓ |
79 lines elided |
↑ open up ↑ |
3378 3379 movq 0x50(%rbx), %rcx
3379 3380 movq %rcx, REGOFF_RFL(%rsp)
3380 3381 movq %rbx, %rcx
3381 3382 addq $0x60, %rcx
3382 3383 movq %rcx, REGOFF_RSP(%rsp)
3383 3384 movw %ss, %cx
3384 3385 movzwq %cx, %rcx
3385 3386 movq %rcx, REGOFF_SS(%rsp)
3386 3387
3387 3388 /*
3388 - * panicsys(format, alist, rp, on_panic_stack)
3389 - */
3389 + * panicsys(format, alist, rp, on_panic_stack)
3390 + */
3390 3391 movq REGOFF_RDI(%rsp), %rdi /* format */
3391 3392 movq REGOFF_RSI(%rsp), %rsi /* alist */
3392 3393 movq %rsp, %rdx /* struct regs */
3393 3394 movl %r11d, %ecx /* on_panic_stack */
3394 3395 call panicsys
3395 3396 addq $REGSIZE, %rsp
3396 3397 popq %rdi
3397 3398 popq %rsi
3398 3399 popq %rdx
3399 3400 popq %rcx
3400 3401 popq %r8
3401 3402 popq %r9
3402 3403 popq %rax
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
3403 3404 popq %rbx
3404 3405 popq %r10
3405 3406 popq %r11
3406 3407 popfq
3407 3408 leave
3408 3409 ret
3409 3410 SET_SIZE(vpanic)
3410 3411
3411 3412 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3412 3413
3413 - pushq %rbp /* | %rip | 0x60 */
3414 + pushq %rbp /* | %rip | 0x60 */
3414 3415 movq %rsp, %rbp /* | %rbp | 0x58 */
3415 3416 pushfq /* | rfl | 0x50 */
3416 3417 pushq %r11 /* | %r11 | 0x48 */
3417 3418 pushq %r10 /* | %r10 | 0x40 */
3418 3419 pushq %rbx /* | %rbx | 0x38 */
3419 3420 pushq %rax /* | %rax | 0x30 */
3420 3421 pushq %r9 /* | %r9 | 0x28 */
3421 3422 pushq %r8 /* | %r8 | 0x20 */
3422 3423 pushq %rcx /* | %rcx | 0x18 */
3423 3424 pushq %rdx /* | %rdx | 0x10 */
3424 3425 pushq %rsi /* | %rsi | 0x8 alist */
3425 3426 pushq %rdi /* | %rdi | 0x0 format */
3426 3427
3427 3428 movq %rsp, %rbx /* %rbx = current %rsp */
3428 3429
3429 3430 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3430 3431 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3431 3432 jmp vpanic_common
3432 3433
3433 3434 SET_SIZE(dtrace_vpanic)
3434 3435
3435 3436 #elif defined(__i386)
3436 3437
3437 3438 ENTRY_NP(vpanic) / Initial stack layout:
3438 3439
3439 3440 pushl %ebp / | %eip | 20
3440 3441 movl %esp, %ebp / | %ebp | 16
3441 3442 pushl %eax / | %eax | 12
3442 3443 pushl %ebx / | %ebx | 8
3443 3444 pushl %ecx / | %ecx | 4
3444 3445 pushl %edx / | %edx | 0
3445 3446
3446 3447 movl %esp, %ebx / %ebx = current stack pointer
3447 3448
3448 3449 lea panic_quiesce, %eax / %eax = &panic_quiesce
3449 3450 pushl %eax / push &panic_quiesce
3450 3451 call panic_trigger / %eax = panic_trigger()
3451 3452 addl $4, %esp / reset stack pointer
3452 3453
3453 3454 vpanic_common:
3454 3455 cmpl $0, %eax / if (%eax == 0)
3455 3456 je 0f / goto 0f;
3456 3457
3457 3458 /*
3458 3459 * If panic_trigger() was successful, we are the first to initiate a
|
↓ open down ↓ |
35 lines elided |
↑ open up ↑ |
3459 3460 * panic: we now switch to the reserved panic_stack before continuing.
3460 3461 */
3461 3462 lea panic_stack, %esp / %esp = panic_stack
3462 3463 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3463 3464
3464 3465 0: subl $REGSIZE, %esp / allocate struct regs
3465 3466
3466 3467 /*
3467 3468 * Now that we've got everything set up, store the register values as
3468 3469 * they were when we entered vpanic() to the designated location in
3469 - * the regs structure we allocated on the stack.
3470 + * the regs structure we allocated on the stack.
3470 3471 */
3471 3472 #if !defined(__GNUC_AS__)
3472 3473 movw %gs, %edx
3473 3474 movl %edx, REGOFF_GS(%esp)
3474 3475 movw %fs, %edx
3475 3476 movl %edx, REGOFF_FS(%esp)
3476 3477 movw %es, %edx
3477 3478 movl %edx, REGOFF_ES(%esp)
3478 3479 movw %ds, %edx
3479 3480 movl %edx, REGOFF_DS(%esp)
3480 3481 #else /* __GNUC_AS__ */
3481 3482 mov %gs, %edx
3482 3483 mov %edx, REGOFF_GS(%esp)
3483 3484 mov %fs, %edx
3484 3485 mov %edx, REGOFF_FS(%esp)
3485 3486 mov %es, %edx
3486 3487 mov %edx, REGOFF_ES(%esp)
3487 3488 mov %ds, %edx
3488 3489 mov %edx, REGOFF_DS(%esp)
3489 3490 #endif /* __GNUC_AS__ */
3490 3491 movl %edi, REGOFF_EDI(%esp)
3491 3492 movl %esi, REGOFF_ESI(%esp)
3492 3493 movl 16(%ebx), %ecx
3493 3494 movl %ecx, REGOFF_EBP(%esp)
3494 3495 movl %ebx, %ecx
3495 3496 addl $20, %ecx
3496 3497 movl %ecx, REGOFF_ESP(%esp)
3497 3498 movl 8(%ebx), %ecx
3498 3499 movl %ecx, REGOFF_EBX(%esp)
3499 3500 movl 0(%ebx), %ecx
3500 3501 movl %ecx, REGOFF_EDX(%esp)
3501 3502 movl 4(%ebx), %ecx
3502 3503 movl %ecx, REGOFF_ECX(%esp)
3503 3504 movl 12(%ebx), %ecx
3504 3505 movl %ecx, REGOFF_EAX(%esp)
3505 3506 movl $0, REGOFF_TRAPNO(%esp)
3506 3507 movl $0, REGOFF_ERR(%esp)
3507 3508 lea vpanic, %ecx
3508 3509 movl %ecx, REGOFF_EIP(%esp)
3509 3510 #if !defined(__GNUC_AS__)
3510 3511 movw %cs, %edx
3511 3512 #else /* __GNUC_AS__ */
3512 3513 mov %cs, %edx
3513 3514 #endif /* __GNUC_AS__ */
3514 3515 movl %edx, REGOFF_CS(%esp)
3515 3516 pushfl
3516 3517 popl %ecx
3517 3518 #if defined(__xpv)
3518 3519 /*
3519 3520 * Synthesize the PS_IE bit from the event mask bit
3520 3521 */
3521 3522 CURTHREAD(%edx)
3522 3523 KPREEMPT_DISABLE(%edx)
3523 3524 EVENT_MASK_TO_IE(%edx, %ecx)
3524 3525 CURTHREAD(%edx)
3525 3526 KPREEMPT_ENABLE_NOKP(%edx)
3526 3527 #endif
3527 3528 movl %ecx, REGOFF_EFL(%esp)
3528 3529 movl $0, REGOFF_UESP(%esp)
3529 3530 #if !defined(__GNUC_AS__)
3530 3531 movw %ss, %edx
3531 3532 #else /* __GNUC_AS__ */
3532 3533 mov %ss, %edx
3533 3534 #endif /* __GNUC_AS__ */
3534 3535 movl %edx, REGOFF_SS(%esp)
3535 3536
3536 3537 movl %esp, %ecx / %ecx = ®s
3537 3538 pushl %eax / push on_panic_stack
3538 3539 pushl %ecx / push ®s
3539 3540 movl 12(%ebp), %ecx / %ecx = alist
3540 3541 pushl %ecx / push alist
3541 3542 movl 8(%ebp), %ecx / %ecx = format
3542 3543 pushl %ecx / push format
3543 3544 call panicsys / panicsys();
3544 3545 addl $16, %esp / pop arguments
3545 3546
3546 3547 addl $REGSIZE, %esp
3547 3548 popl %edx
3548 3549 popl %ecx
3549 3550 popl %ebx
3550 3551 popl %eax
3551 3552 leave
3552 3553 ret
3553 3554 SET_SIZE(vpanic)
3554 3555
3555 3556 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3556 3557
3557 3558 pushl %ebp / | %eip | 20
3558 3559 movl %esp, %ebp / | %ebp | 16
3559 3560 pushl %eax / | %eax | 12
3560 3561 pushl %ebx / | %ebx | 8
3561 3562 pushl %ecx / | %ecx | 4
3562 3563 pushl %edx / | %edx | 0
3563 3564
3564 3565 movl %esp, %ebx / %ebx = current stack pointer
3565 3566
3566 3567 lea panic_quiesce, %eax / %eax = &panic_quiesce
3567 3568 pushl %eax / push &panic_quiesce
3568 3569 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3569 3570 addl $4, %esp / reset stack pointer
3570 3571 jmp vpanic_common / jump back to common code
3571 3572
3572 3573 SET_SIZE(dtrace_vpanic)
3573 3574
3574 3575 #endif /* __i386 */
3575 3576 #endif /* __lint */
3576 3577
3577 3578 #if defined(__lint)
3578 3579
3579 3580 void
3580 3581 hres_tick(void)
3581 3582 {}
3582 3583
3583 3584 int64_t timedelta;
3584 3585 hrtime_t hrtime_base;
3585 3586
3586 3587 #else /* __lint */
3587 3588
3588 3589 DGDEF3(timedelta, 8, 8)
3589 3590 .long 0, 0
3590 3591
3591 3592 /*
3592 3593 * initialized to a non zero value to make pc_gethrtime()
3593 3594 * work correctly even before clock is initialized
3594 3595 */
3595 3596 DGDEF3(hrtime_base, 8, 8)
3596 3597 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3597 3598
3598 3599 DGDEF3(adj_shift, 4, 4)
3599 3600 .long ADJ_SHIFT
3600 3601
3601 3602 #if defined(__amd64)
3602 3603
|
↓ open down ↓ |
123 lines elided |
↑ open up ↑ |
3603 3604 ENTRY_NP(hres_tick)
3604 3605 pushq %rbp
3605 3606 movq %rsp, %rbp
3606 3607
3607 3608 /*
3608 3609 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3609 3610 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3610 3611 * At worst, performing this now instead of under CLOCK_LOCK may
3611 3612 * introduce some jitter in pc_gethrestime().
3612 3613 */
3613 - call *gethrtimef(%rip)
3614 + movq gethrtimef(%rip), %rsi
3615 + INDIRECT_CALL_REG(rsi)
3614 3616 movq %rax, %r8
3615 3617
3616 3618 leaq hres_lock(%rip), %rax
3617 3619 movb $-1, %dl
3618 3620 .CL1:
3619 3621 xchgb %dl, (%rax)
3620 3622 testb %dl, %dl
3621 3623 jz .CL3 /* got it */
3622 3624 .CL2:
3623 3625 cmpb $0, (%rax) /* possible to get lock? */
3624 3626 pause
3625 3627 jne .CL2
3626 3628 jmp .CL1 /* yes, try again */
3627 3629 .CL3:
3628 3630 /*
3629 3631 * compute the interval since last time hres_tick was called
3630 3632 * and adjust hrtime_base and hrestime accordingly
|
↓ open down ↓ |
7 lines elided |
↑ open up ↑ |
3631 3633 * hrtime_base is an 8 byte value (in nsec), hrestime is
3632 3634 * a timestruc_t (sec, nsec)
3633 3635 */
3634 3636 leaq hres_last_tick(%rip), %rax
3635 3637 movq %r8, %r11
3636 3638 subq (%rax), %r8
3637 3639 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3638 3640 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3639 3641 /*
3640 3642 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3641 - */
3642 - movq %r11, (%rax)
3643 + */
3644 + movq %r11, (%rax)
3643 3645
3644 3646 call __adj_hrestime
3645 3647
3646 3648 /*
3647 3649 * release the hres_lock
3648 3650 */
3649 3651 incl hres_lock(%rip)
3650 3652 leave
3651 3653 ret
3652 3654 SET_SIZE(hres_tick)
3653 -
3655 +
3654 3656 #elif defined(__i386)
3655 3657
3656 3658 ENTRY_NP(hres_tick)
3657 3659 pushl %ebp
3658 3660 movl %esp, %ebp
3659 3661 pushl %esi
3660 3662 pushl %ebx
3661 3663
3662 3664 /*
3663 3665 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3664 3666 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3665 3667 * At worst, performing this now instead of under CLOCK_LOCK may
3666 3668 * introduce some jitter in pc_gethrestime().
3667 3669 */
3668 3670 call *gethrtimef
3669 3671 movl %eax, %ebx
3670 3672 movl %edx, %esi
3671 3673
3672 3674 movl $hres_lock, %eax
3673 3675 movl $-1, %edx
3674 3676 .CL1:
3675 3677 xchgb %dl, (%eax)
3676 3678 testb %dl, %dl
3677 3679 jz .CL3 / got it
3678 3680 .CL2:
3679 3681 cmpb $0, (%eax) / possible to get lock?
3680 3682 pause
3681 3683 jne .CL2
3682 3684 jmp .CL1 / yes, try again
3683 3685 .CL3:
3684 3686 /*
3685 3687 * compute the interval since last time hres_tick was called
|
↓ open down ↓ |
22 lines elided |
↑ open up ↑ |
3686 3688 * and adjust hrtime_base and hrestime accordingly
3687 3689 * hrtime_base is an 8 byte value (in nsec), hrestime is
3688 3690 * timestruc_t (sec, nsec)
3689 3691 */
3690 3692
3691 3693 lea hres_last_tick, %eax
3692 3694
3693 3695 movl %ebx, %edx
3694 3696 movl %esi, %ecx
3695 3697
3696 - subl (%eax), %edx
3697 - sbbl 4(%eax), %ecx
3698 + subl (%eax), %edx
3699 + sbbl 4(%eax), %ecx
3698 3700
3699 3701 addl %edx, hrtime_base / add interval to hrtime_base
3700 3702 adcl %ecx, hrtime_base+4
3701 3703
3702 - addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3704 + addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3703 3705
3704 3706 /
3705 3707 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3706 3708 /
3707 3709 movl %ebx, (%eax)
3708 3710 movl %esi, 4(%eax)
3709 3711
3710 3712 / get hrestime at this moment. used as base for pc_gethrestime
3711 3713 /
3712 3714 / Apply adjustment, if any
3713 3715 /
3714 3716 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3715 3717 / (max_hres_adj)
3716 3718 /
3717 3719 / void
3718 3720 / adj_hrestime()
3719 3721 / {
3720 3722 / long long adj;
3721 3723 /
3722 3724 / if (hrestime_adj == 0)
3723 3725 / adj = 0;
3724 3726 / else if (hrestime_adj > 0) {
3725 3727 / if (hrestime_adj < HRES_ADJ)
3726 3728 / adj = hrestime_adj;
3727 3729 / else
3728 3730 / adj = HRES_ADJ;
3729 3731 / }
3730 3732 / else {
3731 3733 / if (hrestime_adj < -(HRES_ADJ))
3732 3734 / adj = -(HRES_ADJ);
3733 3735 / else
3734 3736 / adj = hrestime_adj;
3735 3737 / }
3736 3738 /
3737 3739 / timedelta -= adj;
3738 3740 / hrestime_adj = timedelta;
3739 3741 / hrestime.tv_nsec += adj;
3740 3742 /
3741 3743 / while (hrestime.tv_nsec >= NANOSEC) {
3742 3744 / one_sec++;
3743 3745 / hrestime.tv_sec++;
3744 3746 / hrestime.tv_nsec -= NANOSEC;
3745 3747 / }
3746 3748 / }
3747 3749 __adj_hrestime:
3748 3750 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3749 3751 movl hrestime_adj+4, %edx
3750 3752 andl %esi, %esi
3751 3753 jne .CL4 / no
3752 3754 andl %edx, %edx
3753 3755 jne .CL4 / no
3754 3756 subl %ecx, %ecx / yes, adj = 0;
3755 3757 subl %edx, %edx
3756 3758 jmp .CL5
3757 3759 .CL4:
3758 3760 subl %ecx, %ecx
3759 3761 subl %eax, %eax
3760 3762 subl %esi, %ecx
3761 3763 sbbl %edx, %eax
3762 3764 andl %eax, %eax / if (hrestime_adj > 0)
3763 3765 jge .CL6
3764 3766
3765 3767 / In the following comments, HRES_ADJ is used, while in the code
3766 3768 / max_hres_adj is used.
3767 3769 /
3768 3770 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3769 3771 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3770 3772 / on the logical equivalence of:
3771 3773 /
3772 3774 / !(hrestime_adj < HRES_ADJ)
3773 3775 /
3774 3776 / and the two step sequence:
3775 3777 /
3776 3778 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3777 3779 /
3778 3780 / which computes whether or not the least significant 32-bits
3779 3781 / of hrestime_adj is greater than HRES_ADJ, followed by:
3780 3782 /
3781 3783 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3782 3784 /
3783 3785 / which generates a carry whenever step 1 is true or the most
3784 3786 / significant long of the longlong hrestime_adj is non-zero.
3785 3787
3786 3788 movl max_hres_adj, %ecx / hrestime_adj is positive
3787 3789 subl %esi, %ecx
3788 3790 movl %edx, %eax
3789 3791 adcl $-1, %eax
3790 3792 jnc .CL7
3791 3793 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3792 3794 subl %edx, %edx
3793 3795 jmp .CL5
3794 3796
3795 3797 / The following computation is similar to the one above.
3796 3798 /
3797 3799 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3798 3800 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3799 3801 / on the logical equivalence of:
3800 3802 /
3801 3803 / (hrestime_adj > -HRES_ADJ)
3802 3804 /
3803 3805 / and the two step sequence:
3804 3806 /
3805 3807 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3806 3808 /
3807 3809 / which means the least significant 32-bits of hrestime_adj is
3808 3810 / greater than -HRES_ADJ, followed by:
3809 3811 /
3810 3812 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3811 3813 /
3812 3814 / which generates a carry only when step 1 is true and the most
3813 3815 / significant long of the longlong hrestime_adj is -1.
3814 3816
3815 3817 .CL6: / hrestime_adj is negative
3816 3818 movl %esi, %ecx
3817 3819 addl max_hres_adj, %ecx
3818 3820 movl %edx, %eax
3819 3821 adcl $0, %eax
3820 3822 jc .CL7
3821 3823 xor %ecx, %ecx
3822 3824 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3823 3825 movl $-1, %edx
3824 3826 jmp .CL5
3825 3827 .CL7:
3826 3828 movl %esi, %ecx / adj = hrestime_adj;
3827 3829 .CL5:
3828 3830 movl timedelta, %esi
3829 3831 subl %ecx, %esi
3830 3832 movl timedelta+4, %eax
3831 3833 sbbl %edx, %eax
3832 3834 movl %esi, timedelta
3833 3835 movl %eax, timedelta+4 / timedelta -= adj;
3834 3836 movl %esi, hrestime_adj
3835 3837 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3836 3838 addl hrestime+4, %ecx
3837 3839
3838 3840 movl %ecx, %eax / eax = tv_nsec
3839 3841 1:
3840 3842 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3841 3843 jb .CL8 / no
3842 3844 incl one_sec / yes, one_sec++;
3843 3845 incl hrestime / hrestime.tv_sec++;
3844 3846 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3845 3847 jmp 1b / check for more seconds
3846 3848
3847 3849 .CL8:
3848 3850 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3849 3851 incl hres_lock / release the hres_lock
3850 3852
3851 3853 popl %ebx
3852 3854 popl %esi
3853 3855 leave
3854 3856 ret
3855 3857 SET_SIZE(hres_tick)
3856 3858
3857 3859 #endif /* __i386 */
3858 3860 #endif /* __lint */
3859 3861
3860 3862 /*
3861 3863 * void prefetch_smap_w(void *)
3862 3864 *
3863 3865 * Prefetch ahead within a linear list of smap structures.
3864 3866 * Not implemented for ia32. Stub for compatibility.
3865 3867 */
3866 3868
3867 3869 #if defined(__lint)
3868 3870
3869 3871 /*ARGSUSED*/
3870 3872 void prefetch_smap_w(void *smp)
3871 3873 {}
3872 3874
3873 3875 #else /* __lint */
3874 3876
3875 3877 ENTRY(prefetch_smap_w)
3876 3878 rep; ret /* use 2 byte return instruction when branch target */
3877 3879 /* AMD Software Optimization Guide - Section 6.2 */
3878 3880 SET_SIZE(prefetch_smap_w)
3879 3881
3880 3882 #endif /* __lint */
3881 3883
3882 3884 /*
3883 3885 * prefetch_page_r(page_t *)
3884 3886 * issue prefetch instructions for a page_t
3885 3887 */
3886 3888 #if defined(__lint)
3887 3889
3888 3890 /*ARGSUSED*/
3889 3891 void
3890 3892 prefetch_page_r(void *pp)
3891 3893 {}
3892 3894
3893 3895 #else /* __lint */
3894 3896
3895 3897 ENTRY(prefetch_page_r)
3896 3898 rep; ret /* use 2 byte return instruction when branch target */
3897 3899 /* AMD Software Optimization Guide - Section 6.2 */
3898 3900 SET_SIZE(prefetch_page_r)
3899 3901
3900 3902 #endif /* __lint */
3901 3903
3902 3904 #if defined(__lint)
3903 3905
3904 3906 /*ARGSUSED*/
3905 3907 int
3906 3908 bcmp(const void *s1, const void *s2, size_t count)
3907 3909 { return (0); }
3908 3910
3909 3911 #else /* __lint */
3910 3912
3911 3913 #if defined(__amd64)
3912 3914
3913 3915 ENTRY(bcmp)
3914 3916 pushq %rbp
3915 3917 movq %rsp, %rbp
3916 3918 #ifdef DEBUG
3917 3919 testq %rdx,%rdx
3918 3920 je 1f
3919 3921 movq postbootkernelbase(%rip), %r11
3920 3922 cmpq %r11, %rdi
3921 3923 jb 0f
3922 3924 cmpq %r11, %rsi
3923 3925 jnb 1f
3924 3926 0: leaq .bcmp_panic_msg(%rip), %rdi
3925 3927 xorl %eax, %eax
|
↓ open down ↓ |
213 lines elided |
↑ open up ↑ |
3926 3928 call panic
3927 3929 1:
3928 3930 #endif /* DEBUG */
3929 3931 call memcmp
3930 3932 testl %eax, %eax
3931 3933 setne %dl
3932 3934 leave
3933 3935 movzbl %dl, %eax
3934 3936 ret
3935 3937 SET_SIZE(bcmp)
3936 -
3938 +
3937 3939 #elif defined(__i386)
3938 -
3940 +
3939 3941 #define ARG_S1 8
3940 3942 #define ARG_S2 12
3941 3943 #define ARG_LENGTH 16
3942 3944
3943 3945 ENTRY(bcmp)
3944 3946 pushl %ebp
3945 3947 movl %esp, %ebp / create new stack frame
3946 3948 #ifdef DEBUG
3947 3949 cmpl $0, ARG_LENGTH(%ebp)
3948 3950 je 1f
3949 3951 movl postbootkernelbase, %eax
3950 3952 cmpl %eax, ARG_S1(%ebp)
3951 3953 jb 0f
3952 3954 cmpl %eax, ARG_S2(%ebp)
3953 3955 jnb 1f
3954 3956 0: pushl $.bcmp_panic_msg
3955 3957 call panic
3956 3958 1:
3957 3959 #endif /* DEBUG */
3958 3960
3959 3961 pushl %edi / save register variable
3960 3962 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3961 3963 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3962 3964 cmpl %eax, %ecx / if the same string
3963 3965 je .equal / goto .equal
3964 3966 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3965 3967 cmpl $4, %edi / if %edi < 4
3966 3968 jb .byte_check / goto .byte_check
3967 3969 .align 4
3968 3970 .word_loop:
3969 3971 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3970 3972 leal -4(%edi), %edi / %edi -= 4
3971 3973 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3972 3974 jne .word_not_equal / if not equal, goto .word_not_equal
3973 3975 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3974 3976 leal 4(%eax), %eax / %eax += 4 (next word)
3975 3977 cmpl $4, %edi / if %edi >= 4
3976 3978 jae .word_loop / goto .word_loop
3977 3979 .byte_check:
3978 3980 cmpl $0, %edi / if %edi == 0
3979 3981 je .equal / goto .equal
3980 3982 jmp .byte_loop / goto .byte_loop (checks in bytes)
3981 3983 .word_not_equal:
3982 3984 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3983 3985 .align 4
3984 3986 .byte_loop:
3985 3987 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3986 3988 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3987 3989 jne .not_equal / if not equal, goto .not_equal
3988 3990 incl %ecx / %ecx++ (next byte)
3989 3991 incl %eax / %eax++ (next byte)
3990 3992 decl %edi / %edi--
3991 3993 jnz .byte_loop / if not zero, goto .byte_loop
3992 3994 .equal:
3993 3995 xorl %eax, %eax / %eax = 0
3994 3996 popl %edi / restore register variable
3995 3997 leave / restore old stack frame
3996 3998 ret / return (NULL)
3997 3999 .align 4
3998 4000 .not_equal:
3999 4001 movl $1, %eax / return 1
4000 4002 popl %edi / restore register variable
4001 4003 leave / restore old stack frame
4002 4004 ret / return (NULL)
4003 4005 SET_SIZE(bcmp)
4004 4006
4005 4007 #endif /* __i386 */
4006 4008
4007 4009 #ifdef DEBUG
4008 4010 .text
4009 4011 .bcmp_panic_msg:
4010 4012 .string "bcmp: arguments below kernelbase"
4011 4013 #endif /* DEBUG */
4012 4014
4013 4015 #endif /* __lint */
4014 4016
4015 4017 #if defined(__lint)
4016 4018
4017 4019 uint_t
4018 4020 bsrw_insn(uint16_t mask)
4019 4021 {
4020 4022 uint_t index = sizeof (mask) * NBBY - 1;
4021 4023
4022 4024 while ((mask & (1 << index)) == 0)
4023 4025 index--;
4024 4026 return (index);
4025 4027 }
4026 4028
4027 4029 #else /* __lint */
4028 4030
4029 4031 #if defined(__amd64)
4030 4032
4031 4033 ENTRY_NP(bsrw_insn)
4032 4034 xorl %eax, %eax
4033 4035 bsrw %di, %ax
4034 4036 ret
4035 4037 SET_SIZE(bsrw_insn)
4036 4038
4037 4039 #elif defined(__i386)
4038 4040
4039 4041 ENTRY_NP(bsrw_insn)
4040 4042 movw 4(%esp), %cx
4041 4043 xorl %eax, %eax
4042 4044 bsrw %cx, %ax
4043 4045 ret
4044 4046 SET_SIZE(bsrw_insn)
4045 4047
4046 4048 #endif /* __i386 */
4047 4049 #endif /* __lint */
4048 4050
4049 4051 #if defined(__lint)
4050 4052
4051 4053 uint_t
4052 4054 atomic_btr32(uint32_t *pending, uint_t pil)
4053 4055 {
4054 4056 return (*pending &= ~(1 << pil));
4055 4057 }
4056 4058
4057 4059 #else /* __lint */
4058 4060
4059 4061 #if defined(__i386)
4060 4062
4061 4063 ENTRY_NP(atomic_btr32)
4062 4064 movl 4(%esp), %ecx
4063 4065 movl 8(%esp), %edx
4064 4066 xorl %eax, %eax
4065 4067 lock
4066 4068 btrl %edx, (%ecx)
4067 4069 setc %al
4068 4070 ret
4069 4071 SET_SIZE(atomic_btr32)
4070 4072
4071 4073 #endif /* __i386 */
4072 4074 #endif /* __lint */
4073 4075
4074 4076 #if defined(__lint)
4075 4077
4076 4078 /*ARGSUSED*/
4077 4079 void
4078 4080 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4079 4081 uint_t arg2)
4080 4082 {}
4081 4083
4082 4084 #else /* __lint */
|
↓ open down ↓ |
134 lines elided |
↑ open up ↑ |
4083 4085
4084 4086 #if defined(__amd64)
4085 4087
4086 4088 ENTRY_NP(switch_sp_and_call)
4087 4089 pushq %rbp
4088 4090 movq %rsp, %rbp /* set up stack frame */
4089 4091 movq %rdi, %rsp /* switch stack pointer */
4090 4092 movq %rdx, %rdi /* pass func arg 1 */
4091 4093 movq %rsi, %r11 /* save function to call */
4092 4094 movq %rcx, %rsi /* pass func arg 2 */
4093 - call *%r11 /* call function */
4095 + INDIRECT_CALL_REG(r11) /* call function */
4094 4096 leave /* restore stack */
4095 4097 ret
4096 4098 SET_SIZE(switch_sp_and_call)
4097 4099
4098 4100 #elif defined(__i386)
4099 4101
4100 4102 ENTRY_NP(switch_sp_and_call)
4101 4103 pushl %ebp
4102 4104 mov %esp, %ebp /* set up stack frame */
4103 4105 movl 8(%ebp), %esp /* switch stack pointer */
4104 4106 pushl 20(%ebp) /* push func arg 2 */
4105 4107 pushl 16(%ebp) /* push func arg 1 */
4106 4108 call *12(%ebp) /* call function */
4107 4109 addl $8, %esp /* pop arguments */
4108 4110 leave /* restore stack */
4109 4111 ret
4110 4112 SET_SIZE(switch_sp_and_call)
4111 4113
4112 4114 #endif /* __i386 */
4113 4115 #endif /* __lint */
4114 4116
4115 4117 #if defined(__lint)
4116 4118
4117 4119 void
4118 4120 kmdb_enter(void)
4119 4121 {}
4120 4122
4121 4123 #else /* __lint */
4122 4124
4123 4125 #if defined(__amd64)
4124 4126
4125 4127 ENTRY_NP(kmdb_enter)
4126 4128 pushq %rbp
4127 4129 movq %rsp, %rbp
4128 4130
4129 4131 /*
4130 4132 * Save flags, do a 'cli' then return the saved flags
4131 4133 */
4132 4134 call intr_clear
|
↓ open down ↓ |
29 lines elided |
↑ open up ↑ |
4133 4135
4134 4136 int $T_DBGENTR
4135 4137
4136 4138 /*
4137 4139 * Restore the saved flags
4138 4140 */
4139 4141 movq %rax, %rdi
4140 4142 call intr_restore
4141 4143
4142 4144 leave
4143 - ret
4145 + ret
4144 4146 SET_SIZE(kmdb_enter)
4145 4147
4146 4148 #elif defined(__i386)
4147 4149
4148 4150 ENTRY_NP(kmdb_enter)
4149 4151 pushl %ebp
4150 4152 movl %esp, %ebp
4151 4153
4152 4154 /*
4153 4155 * Save flags, do a 'cli' then return the saved flags
4154 4156 */
4155 4157 call intr_clear
4156 4158
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
4157 4159 int $T_DBGENTR
4158 4160
4159 4161 /*
4160 4162 * Restore the saved flags
4161 4163 */
4162 4164 pushl %eax
4163 4165 call intr_restore
4164 4166 addl $4, %esp
4165 4167
4166 4168 leave
4167 - ret
4169 + ret
4168 4170 SET_SIZE(kmdb_enter)
4169 4171
4170 4172 #endif /* __i386 */
4171 4173 #endif /* __lint */
4172 4174
4173 4175 #if defined(__lint)
4174 4176
4175 4177 void
4176 4178 return_instr(void)
4177 4179 {}
4178 4180
4179 4181 #else /* __lint */
4180 4182
4181 4183 ENTRY_NP(return_instr)
4182 4184 rep; ret /* use 2 byte instruction when branch target */
4183 4185 /* AMD Software Optimization Guide - Section 6.2 */
4184 4186 SET_SIZE(return_instr)
4185 4187
4186 4188 #endif /* __lint */
4187 4189
4188 4190 #if defined(__lint)
4189 4191
4190 4192 ulong_t
4191 4193 getflags(void)
4192 4194 {
4193 4195 return (0);
4194 4196 }
4195 4197
4196 4198 #else /* __lint */
4197 4199
4198 4200 #if defined(__amd64)
4199 4201
4200 4202 ENTRY(getflags)
4201 4203 pushfq
4202 4204 popq %rax
4203 4205 #if defined(__xpv)
4204 4206 CURTHREAD(%rdi)
4205 4207 KPREEMPT_DISABLE(%rdi)
4206 4208 /*
4207 4209 * Synthesize the PS_IE bit from the event mask bit
4208 4210 */
4209 4211 CURVCPU(%r11)
4210 4212 andq $_BITNOT(PS_IE), %rax
4211 4213 XEN_TEST_UPCALL_MASK(%r11)
4212 4214 jnz 1f
4213 4215 orq $PS_IE, %rax
4214 4216 1:
4215 4217 KPREEMPT_ENABLE_NOKP(%rdi)
4216 4218 #endif
4217 4219 ret
4218 4220 SET_SIZE(getflags)
4219 4221
4220 4222 #elif defined(__i386)
4221 4223
4222 4224 ENTRY(getflags)
4223 4225 pushfl
4224 4226 popl %eax
4225 4227 #if defined(__xpv)
4226 4228 CURTHREAD(%ecx)
4227 4229 KPREEMPT_DISABLE(%ecx)
4228 4230 /*
4229 4231 * Synthesize the PS_IE bit from the event mask bit
4230 4232 */
4231 4233 CURVCPU(%edx)
4232 4234 andl $_BITNOT(PS_IE), %eax
4233 4235 XEN_TEST_UPCALL_MASK(%edx)
4234 4236 jnz 1f
4235 4237 orl $PS_IE, %eax
4236 4238 1:
4237 4239 KPREEMPT_ENABLE_NOKP(%ecx)
4238 4240 #endif
4239 4241 ret
4240 4242 SET_SIZE(getflags)
4241 4243
4242 4244 #endif /* __i386 */
4243 4245
4244 4246 #endif /* __lint */
4245 4247
4246 4248 #if defined(__lint)
4247 4249
4248 4250 ftrace_icookie_t
4249 4251 ftrace_interrupt_disable(void)
4250 4252 { return (0); }
4251 4253
4252 4254 #else /* __lint */
4253 4255
|
↓ open down ↓ |
76 lines elided |
↑ open up ↑ |
4254 4256 #if defined(__amd64)
4255 4257
4256 4258 ENTRY(ftrace_interrupt_disable)
4257 4259 pushfq
4258 4260 popq %rax
4259 4261 CLI(%rdx)
4260 4262 ret
4261 4263 SET_SIZE(ftrace_interrupt_disable)
4262 4264
4263 4265 #elif defined(__i386)
4264 -
4266 +
4265 4267 ENTRY(ftrace_interrupt_disable)
4266 4268 pushfl
4267 4269 popl %eax
4268 4270 CLI(%edx)
4269 4271 ret
4270 4272 SET_SIZE(ftrace_interrupt_disable)
4271 4273
4272 -#endif /* __i386 */
4274 +#endif /* __i386 */
4273 4275 #endif /* __lint */
4274 4276
4275 4277 #if defined(__lint)
4276 4278
4277 4279 /*ARGSUSED*/
4278 4280 void
4279 4281 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4280 4282 {}
4281 4283
4282 4284 #else /* __lint */
4283 4285
4284 4286 #if defined(__amd64)
4285 4287
4286 4288 ENTRY(ftrace_interrupt_enable)
4287 4289 pushq %rdi
4288 4290 popfq
4289 4291 ret
4290 4292 SET_SIZE(ftrace_interrupt_enable)
4291 4293
4292 4294 #elif defined(__i386)
4293 -
4295 +
4294 4296 ENTRY(ftrace_interrupt_enable)
4295 4297 movl 4(%esp), %eax
4296 4298 pushl %eax
4297 4299 popfl
4298 4300 ret
4299 4301 SET_SIZE(ftrace_interrupt_enable)
4300 4302
4301 4303 #endif /* __i386 */
4302 4304 #endif /* __lint */
4303 4305
4304 4306 #if defined (__lint)
4305 4307
4306 4308 /*ARGSUSED*/
4307 4309 void
4308 4310 clflush_insn(caddr_t addr)
4309 4311 {}
4310 4312
4311 4313 #else /* __lint */
4312 4314
4313 4315 #if defined (__amd64)
4314 4316 ENTRY(clflush_insn)
4315 4317 clflush (%rdi)
4316 4318 ret
4317 4319 SET_SIZE(clflush_insn)
4318 4320 #elif defined (__i386)
4319 4321 ENTRY(clflush_insn)
4320 4322 movl 4(%esp), %eax
4321 4323 clflush (%eax)
4322 4324 ret
4323 4325 SET_SIZE(clflush_insn)
4324 4326
4325 4327 #endif /* __i386 */
4326 4328 #endif /* __lint */
4327 4329
4328 4330 #if defined (__lint)
4329 4331 /*ARGSUSED*/
4330 4332 void
4331 4333 mfence_insn(void)
4332 4334 {}
4333 4335
4334 4336 #else /* __lint */
4335 4337
4336 4338 #if defined (__amd64)
4337 4339 ENTRY(mfence_insn)
4338 4340 mfence
4339 4341 ret
4340 4342 SET_SIZE(mfence_insn)
4341 4343 #elif defined (__i386)
4342 4344 ENTRY(mfence_insn)
4343 4345 mfence
4344 4346 ret
4345 4347 SET_SIZE(mfence_insn)
|
↓ open down ↓ |
42 lines elided |
↑ open up ↑ |
4346 4348
4347 4349 #endif /* __i386 */
4348 4350 #endif /* __lint */
4349 4351
4350 4352 /*
4351 4353 * VMware implements an I/O port that programs can query to detect if software
4352 4354 * is running in a VMware hypervisor. This hypervisor port behaves differently
4353 4355 * depending on magic values in certain registers and modifies some registers
4354 4356 * as a side effect.
4355 4357 *
4356 - * References: http://kb.vmware.com/kb/1009458
4358 + * References: http://kb.vmware.com/kb/1009458
4357 4359 */
4358 4360
4359 4361 #if defined(__lint)
4360 4362
4361 4363 /* ARGSUSED */
4362 4364 void
4363 4365 vmware_port(int cmd, uint32_t *regs) { return; }
4364 4366
4365 4367 #else
4366 4368
4367 4369 #if defined(__amd64)
4368 4370
4369 4371 ENTRY(vmware_port)
4370 4372 pushq %rbx
4371 4373 movl $VMWARE_HVMAGIC, %eax
4372 4374 movl $0xffffffff, %ebx
4373 4375 movl %edi, %ecx
4374 4376 movl $VMWARE_HVPORT, %edx
4375 4377 inl (%dx)
4376 4378 movl %eax, (%rsi)
4377 4379 movl %ebx, 4(%rsi)
4378 4380 movl %ecx, 8(%rsi)
4379 4381 movl %edx, 12(%rsi)
4380 4382 popq %rbx
4381 4383 ret
4382 4384 SET_SIZE(vmware_port)
4383 4385
4384 4386 #elif defined(__i386)
4385 4387
4386 4388 ENTRY(vmware_port)
4387 4389 pushl %ebx
4388 4390 pushl %esi
4389 4391 movl $VMWARE_HVMAGIC, %eax
4390 4392 movl $0xffffffff, %ebx
4391 4393 movl 12(%esp), %ecx
4392 4394 movl $VMWARE_HVPORT, %edx
4393 4395 inl (%dx)
4394 4396 movl 16(%esp), %esi
4395 4397 movl %eax, (%esi)
4396 4398 movl %ebx, 4(%esi)
4397 4399 movl %ecx, 8(%esi)
4398 4400 movl %edx, 12(%esi)
4399 4401 popl %esi
4400 4402 popl %ebx
4401 4403 ret
4402 4404 SET_SIZE(vmware_port)
4403 4405
4404 4406 #endif /* __i386 */
4405 4407 #endif /* __lint */
|
↓ open down ↓ |
39 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX