Print this page
11787 Kernel needs to be built with retpolines
11788 Kernel needs to generally use RSB stuffing
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: John Levon <john.levon@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/ml/locore.s
+++ new/usr/src/uts/i86pc/ml/locore.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25 /*
26 26 * Copyright 2019 Joyent, Inc.
27 27 */
28 28
29 29 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
30 30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
31 31 /* All Rights Reserved */
32 32
33 33 /* Copyright (c) 1987, 1988 Microsoft Corporation */
34 34 /* All Rights Reserved */
35 35
36 36
37 37 #include <sys/asm_linkage.h>
38 38 #include <sys/asm_misc.h>
39 39 #include <sys/regset.h>
40 40 #include <sys/privregs.h>
41 41 #include <sys/psw.h>
42 42 #include <sys/reboot.h>
43 43 #include <sys/x86_archext.h>
44 44 #include <sys/machparam.h>
45 45
46 46 #if defined(__lint)
47 47
48 48 #include <sys/types.h>
49 49 #include <sys/thread.h>
50 50 #include <sys/systm.h>
51 51 #include <sys/lgrp.h>
52 52 #include <sys/regset.h>
53 53 #include <sys/link.h>
54 54 #include <sys/bootconf.h>
55 55 #include <sys/bootsvcs.h>
56 56
57 57 #else /* __lint */
58 58
59 59 #include <sys/segments.h>
60 60 #include <sys/pcb.h>
61 61 #include <sys/trap.h>
62 62 #include <sys/ftrace.h>
63 63 #include <sys/traptrace.h>
64 64 #include <sys/clock.h>
65 65 #include <sys/cmn_err.h>
66 66 #include <sys/pit.h>
67 67 #include <sys/panic.h>
68 68
69 69 #if defined(__xpv)
70 70 #include <sys/hypervisor.h>
71 71 #endif
72 72
73 73 #include "assym.h"
74 74
75 75 /*
76 76 * Our assumptions:
77 77 * - We are running in protected-paged mode.
78 78 * - Interrupts are disabled.
79 79 * - The GDT and IDT are the callers; we need our copies.
80 80 * - The kernel's text, initialized data and bss are mapped.
81 81 *
82 82 * Our actions:
83 83 * - Save arguments
84 84 * - Initialize our stack pointer to the thread 0 stack (t0stack)
85 85 * and leave room for a phony "struct regs".
86 86 * - Our GDT and IDT need to get munged.
87 87 * - Since we are using the boot's GDT descriptors, we need
88 88 * to copy them into our GDT before we switch to ours.
89 89 * - We start using our GDT by loading correct values in the
90 90 * selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
91 91 * gs=KGS_SEL).
92 92 * - The default LDT entry for syscall is set.
93 93 * - We load the default LDT into the hardware LDT register.
94 94 * - We load the default TSS into the hardware task register.
95 95 * - Check for cpu type, i.e. 486 vs. P5 vs. P6 etc.
96 96 * - mlsetup(%esp) gets called.
97 97 * - We change our appearance to look like the real thread 0.
98 98 * (NOTE: making ourselves to be a real thread may be a noop)
99 99 * - main() gets called. (NOTE: main() never returns).
100 100 *
101 101 * NOW, the real code!
102 102 */
103 103 /*
104 104 * The very first thing in the kernel's text segment must be a jump
105 105 * to the os/fakebop.c startup code.
106 106 */
107 107 .text
108 108 jmp _start
109 109
110 110 /*
111 111 * Globals:
112 112 */
113 113 .globl _locore_start
114 114 .globl mlsetup
115 115 .globl main
116 116 .globl panic
117 117 .globl t0stack
118 118 .globl t0
119 119 .globl sysp
120 120 .globl edata
121 121
122 122 /*
123 123 * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
124 124 */
125 125 .globl bootops
126 126 .globl bootopsp
127 127
128 128 /*
129 129 * NOTE: t0stack should be the first thing in the data section so that
130 130 * if it ever overflows, it will fault on the last kernel text page.
131 131 */
132 132 .data
133 133 .comm t0stack, DEFAULTSTKSZ, 32
134 134 .comm t0, 4094, 32
135 135
136 136 #endif /* __lint */
137 137
138 138
139 139 #if defined(__amd64)
140 140
141 141 #if defined(__lint)
142 142
143 143 /* ARGSUSED */
144 144 void
145 145 _locore_start(struct boot_syscalls *sysp, ulong_t rsi, struct bootops *bop)
146 146 {}
147 147
148 148 #else /* __lint */
149 149
150 150 /*
151 151 * kobj_init() vectors us back to here with (note) a slightly different
152 152 * set of arguments than _start is given (see lint prototypes above).
153 153 *
154 154 * XXX Make this less vile, please.
155 155 */
156 156 ENTRY_NP(_locore_start)
157 157
158 158 /*
159 159 * %rdi = boot services (should die someday)
160 160 * %rdx = bootops
161 161 * end
162 162 */
163 163
164 164 leaq edata(%rip), %rbp /* reference edata for ksyms */
165 165 movq $0, (%rbp) /* limit stack back trace */
166 166
167 167 /*
168 168 * Initialize our stack pointer to the thread 0 stack (t0stack)
169 169 * and leave room for a "struct regs" for lwp0. Note that the
170 170 * stack doesn't actually align to a 16-byte boundary until just
171 171 * before we call mlsetup because we want to use %rsp to point at
172 172 * our regs structure.
173 173 */
174 174 leaq t0stack(%rip), %rsp
175 175 addq $_CONST(DEFAULTSTKSZ - REGSIZE), %rsp
176 176 #if (REGSIZE & 15) == 0
177 177 subq $8, %rsp
178 178 #endif
179 179 /*
180 180 * Save call back for special x86 boot services vector
181 181 */
182 182 movq %rdi, sysp(%rip)
183 183
184 184 movq %rdx, bootops(%rip) /* save bootops */
185 185 movq $bootops, bootopsp(%rip)
186 186
187 187 /*
188 188 * Save arguments and flags, if only for debugging ..
189 189 */
190 190 movq %rdi, REGOFF_RDI(%rsp)
191 191 movq %rsi, REGOFF_RSI(%rsp)
192 192 movq %rdx, REGOFF_RDX(%rsp)
193 193 movq %rcx, REGOFF_RCX(%rsp)
194 194 movq %r8, REGOFF_R8(%rsp)
195 195 movq %r9, REGOFF_R9(%rsp)
196 196 pushf
197 197 popq %r11
198 198 movq %r11, REGOFF_RFL(%rsp)
199 199
200 200 #if !defined(__xpv)
201 201 /*
202 202 * Enable write protect and alignment check faults.
203 203 */
204 204 movq %cr0, %rax
205 205 orq $_CONST(CR0_WP|CR0_AM), %rax
206 206 andq $_BITNOT(CR0_WT|CR0_CE), %rax
207 207 movq %rax, %cr0
208 208 #endif /* __xpv */
209 209
210 210 /*
211 211 * (We just assert this works by virtue of being here)
212 212 */
213 213 bts $X86FSET_CPUID, x86_featureset(%rip)
214 214
215 215 /*
216 216 * mlsetup() gets called with a struct regs as argument, while
217 217 * main takes no args and should never return.
218 218 */
219 219 xorl %ebp, %ebp
220 220 movq %rsp, %rdi
221 221 pushq %rbp
222 222 /* (stack pointer now aligned on 16-byte boundary right here) */
223 223 movq %rsp, %rbp
224 224 call mlsetup
225 225 call main
226 226 /* NOTREACHED */
227 227 leaq __return_from_main(%rip), %rdi
228 228 xorl %eax, %eax
229 229 call panic
230 230 SET_SIZE(_locore_start)
231 231
232 232 #endif /* __amd64 */
233 233 #endif /* __lint */
234 234
235 235 #if !defined(__lint)
236 236
237 237 __return_from_main:
238 238 .string "main() returned"
239 239 __unsupported_cpu:
240 240 .string "486 style cpu detected - no longer supported!"
241 241
242 242 #if defined(DEBUG)
243 243 _no_pending_updates:
244 244 .string "locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
245 245 #endif
246 246
247 247 #endif /* !__lint */
248 248
249 249 #if !defined(__amd64)
250 250
251 251 #if defined(__lint)
252 252
253 253 /* ARGSUSED */
254 254 void
255 255 _locore_start(struct boot_syscalls *sysp, struct bootops *bop)
256 256 {}
257 257
258 258 #else /* __lint */
259 259
260 260 /*
261 261 * kobj_init() vectors us back to here with (note) a slightly different
262 262 * set of arguments than _start is given (see lint prototypes above).
263 263 *
264 264 * XXX Make this less vile, please.
265 265 */
266 266 ENTRY_NP(_locore_start)
267 267
268 268 /*
269 269 * %ecx = boot services (should die someday)
270 270 * %ebx = bootops
271 271 */
272 272 mov $edata, %ebp / edata needs to be defined for ksyms
273 273 movl $0, (%ebp) / limit stack back trace
274 274
275 275 /*
276 276 * Initialize our stack pointer to the thread 0 stack (t0stack)
277 277 * and leave room for a phony "struct regs".
278 278 */
279 279 movl $t0stack + DEFAULTSTKSZ - REGSIZE, %esp
280 280
281 281 /*
282 282 * Save call back for special x86 boot services vector
283 283 */
284 284 mov %ecx, sysp / save call back for boot services
285 285
286 286 mov %ebx, bootops / save bootops
287 287 movl $bootops, bootopsp
288 288
289 289
290 290 /*
291 291 * Save all registers and flags
292 292 */
293 293 pushal
294 294 pushfl
295 295
296 296 #if !defined(__xpv)
297 297 /*
298 298 * Override bios settings and enable write protect and
299 299 * alignment check faults.
300 300 */
301 301 movl %cr0, %eax
302 302
303 303 /*
304 304 * enable WP for detecting faults, and enable alignment checking.
305 305 */
306 306 orl $_CONST(CR0_WP|CR0_AM), %eax
307 307 andl $_BITNOT(CR0_WT|CR0_CE), %eax
308 308 movl %eax, %cr0 / set the cr0 register correctly and
309 309 / override the BIOS setup
310 310
311 311 /*
312 312 * If bit 21 of eflags can be flipped, then cpuid is present
313 313 * and enabled.
314 314 */
315 315 pushfl
316 316 popl %ecx
317 317 movl %ecx, %eax
318 318 xorl $PS_ID, %eax / try complemented bit
319 319 pushl %eax
320 320 popfl
321 321 pushfl
322 322 popl %eax
323 323 cmpl %eax, %ecx
324 324 jne have_cpuid
325 325
326 326 /*
327 327 * cpuid may be disabled on Cyrix, try to detect Cyrix by the 5/2 test
328 328 * div does not modify the cc flags on Cyrix, even though this may
329 329 * also be true for other vendors, this is generally true only for
330 330 * newer models from those vendors that support and do not disable
331 331 * cpuid (usually because cpuid cannot be disabled)
332 332 */
333 333
334 334 /*
335 335 * clear cc flags
336 336 */
337 337 xorb %ah, %ah
338 338 sahf
339 339
340 340 /*
341 341 * perform 5/2 test
342 342 */
343 343 movw $5, %ax
344 344 movb $2, %bl
345 345 divb %bl
346 346
347 347 lahf
348 348 cmpb $2, %ah
349 349 jne cpu_486
350 350
351 351 /*
352 352 * div did not modify the cc flags, chances are the vendor is Cyrix
353 353 * assume the vendor is Cyrix and use the CCR's to enable cpuid
354 354 */
355 355 .set CYRIX_CRI, 0x22 / CR Index Register
356 356 .set CYRIX_CRD, 0x23 / CR Data Register
357 357
358 358 .set CYRIX_CCR3, 0xc3 / Config Control Reg 3
359 359 .set CYRIX_CCR4, 0xe8 / Config Control Reg 4
360 360 .set CYRIX_DIR0, 0xfe / Device Identification Reg 0
361 361 .set CYRIX_DIR1, 0xff / Device Identification Reg 1
362 362
363 363 /*
364 364 * even if the cpu vendor is Cyrix and the motherboard/chipset
365 365 * vendor decided to ignore lines A1-A4 for I/O addresses, I/O port
366 366 * 0x21 corresponds with 0x23 and since 0x22 is still untouched,
367 367 * the reads and writes of 0x21 are guaranteed to be off-chip of
368 368 * the cpu
369 369 */
370 370
371 371 /*
372 372 * enable read of ISR at I/O port 0x20
373 373 */
374 374 movb $0xb, %al
375 375 outb $MCMD_PORT
376 376
377 377 /*
378 378 * read IMR and store in %bl
379 379 */
380 380 inb $MIMR_PORT
381 381 movb %al, %bl
382 382
383 383 /*
384 384 * mask out all interrupts so that ISR will not change
385 385 */
386 386 movb $0xff, %al
387 387 outb $MIMR_PORT
388 388
389 389 /*
390 390 * reads of I/O port 0x22 on Cyrix are always directed off-chip
391 391 * make use of I/O pull-up to test for an unknown device on 0x22
392 392 */
393 393 inb $CYRIX_CRI
394 394 cmpb $0xff, %al
395 395 je port_22_free
396 396
397 397 /*
398 398 * motherboard/chipset vendor may be ignoring line A1 of I/O address
399 399 */
400 400 movb %al, %cl
401 401
402 402 /*
403 403 * if the ISR and the value read from 0x22 do not match then we have
404 404 * detected some unknown device, probably a chipset, at 0x22
405 405 */
406 406 inb $MCMD_PORT
407 407 cmpb %al, %cl
408 408 jne restore_IMR
409 409
410 410 port_22_free:
411 411 /*
412 412 * now test to see if some unknown device is using I/O port 0x23
413 413 *
414 414 * read the external I/O port at 0x23
415 415 */
416 416 inb $CYRIX_CRD
417 417
418 418 /*
419 419 * Test for pull-up at 0x23 or if I/O address line A1 is being ignored.
420 420 * IMR is 0xff so both tests are performed simultaneously.
421 421 */
422 422 cmpb $0xff, %al
423 423 jne restore_IMR
424 424
425 425 /*
426 426 * We are a Cyrix part. In case we are some model of Cx486 or a Cx586,
427 427 * record the type and fix it later if not.
428 428 */
429 429 movl $X86_VENDOR_Cyrix, x86_vendor
430 430 movl $X86_TYPE_CYRIX_486, x86_type
431 431
432 432 /*
433 433 * Try to read CCR3. All Cyrix cpu's which support cpuid have CCR3.
434 434 *
435 435 * load CCR3 index into CCR index register
436 436 */
437 437
438 438 movb $CYRIX_CCR3, %al
439 439 outb $CYRIX_CRI
440 440
441 441 /*
442 442 * If we are not a Cyrix cpu, then we have performed an external I/O
443 443 * cycle. If the CCR index was not valid for this Cyrix model, we may
444 444 * have performed an external I/O cycle as well. In these cases and
445 445 * if the motherboard/chipset vendor ignores I/O address line A1,
446 446 * then the PIC will have IRQ3 set at the lowest priority as a side
447 447 * effect of the above outb. We are reasonalbly confident that there
448 448 * is not an unknown device on I/O port 0x22, so there should have been
449 449 * no unpredictable side-effect of the above outb.
450 450 */
451 451
452 452 /*
453 453 * read CCR3
454 454 */
455 455 inb $CYRIX_CRD
456 456
457 457 /*
458 458 * If we are not a Cyrix cpu the inb above produced an external I/O
459 459 * cycle. If we are a Cyrix model that does not support CCR3 wex
460 460 * produced an external I/O cycle. In all known Cyrix models 6x86 and
461 461 * above, bit 3 of CCR3 is reserved and cannot be set to 1. In all
462 462 * Cyrix models prior to the 6x86 that supported CCR3, bits 4-7 are
463 463 * reserved as well. It is highly unlikely that CCR3 contains the value
464 464 * 0xff. We test to see if I/O port 0x23 is pull-up or the IMR and
465 465 * deduce we are not a Cyrix with support for cpuid if so.
466 466 */
467 467 cmpb $0xff, %al
468 468 je restore_PIC
469 469
470 470 /*
471 471 * There exist 486 ISA Cyrix chips that support CCR3 but do not support
472 472 * DIR0 and DIR1. If we try to read DIR0, we may generate external I/O
473 473 * cycles, the exact behavior is model specific and undocumented.
474 474 * Unfortunately these external I/O cycles may confuse some PIC's beyond
475 475 * recovery. Fortunatetly we can use the following undocumented trick:
476 476 * if bit 4 of CCR3 can be toggled, then DIR0 and DIR1 are supported.
477 477 * Pleasantly MAPEN contains bit 4 of CCR3, so this trick is guaranteed
478 478 * to work on all Cyrix cpu's which support cpuid.
479 479 */
480 480 movb %al, %dl
481 481 xorb $0x10, %dl
482 482 movb %al, %cl
483 483
484 484 /*
485 485 * write back CRR3 with toggled bit 4 to CCR3
486 486 */
487 487 movb $CYRIX_CCR3, %al
488 488 outb $CYRIX_CRI
489 489
490 490 movb %dl, %al
491 491 outb $CYRIX_CRD
492 492
493 493 /*
494 494 * read CCR3
495 495 */
496 496 movb $CYRIX_CCR3, %al
497 497 outb $CYRIX_CRI
498 498 inb $CYRIX_CRD
499 499 movb %al, %dl
500 500
501 501 /*
502 502 * restore CCR3
503 503 */
504 504 movb $CYRIX_CCR3, %al
505 505 outb $CYRIX_CRI
506 506
507 507 movb %cl, %al
508 508 outb $CYRIX_CRD
509 509
510 510 /*
511 511 * if bit 4 was not toggled DIR0 and DIR1 are not supported in which
512 512 * case we do not have cpuid anyway
513 513 */
514 514 andb $0x10, %al
515 515 andb $0x10, %dl
516 516 cmpb %al, %dl
517 517 je restore_PIC
518 518
519 519 /*
520 520 * read DIR0
521 521 */
522 522 movb $CYRIX_DIR0, %al
523 523 outb $CYRIX_CRI
524 524 inb $CYRIX_CRD
525 525
526 526 /*
527 527 * test for pull-up
528 528 */
529 529 cmpb $0xff, %al
530 530 je restore_PIC
531 531
532 532 /*
533 533 * Values of 0x20-0x27 in DIR0 are currently reserved by Cyrix for
534 534 * future use. If Cyrix ever produces a cpu that supports cpuid with
535 535 * these ids, the following test will have to change. For now we remain
536 536 * pessimistic since the formats of the CRR's may be different then.
537 537 *
538 538 * test for at least a 6x86, to see if we support both MAPEN and CPUID
539 539 */
540 540 cmpb $0x30, %al
541 541 jb restore_IMR
542 542
543 543 /*
544 544 * enable MAPEN
545 545 */
546 546 movb $CYRIX_CCR3, %al
547 547 outb $CYRIX_CRI
548 548
549 549 andb $0xf, %cl
550 550 movb %cl, %al
551 551 orb $0x10, %al
552 552 outb $CYRIX_CRD
553 553
554 554 /*
555 555 * select CCR4
556 556 */
557 557 movb $CYRIX_CCR4, %al
558 558 outb $CYRIX_CRI
559 559
560 560 /*
561 561 * read CCR4
562 562 */
563 563 inb $CYRIX_CRD
564 564
565 565 /*
566 566 * enable cpuid
567 567 */
568 568 orb $0x80, %al
569 569 movb %al, %dl
570 570
571 571 /*
572 572 * select CCR4
573 573 */
574 574 movb $CYRIX_CCR4, %al
575 575 outb $CYRIX_CRI
576 576
577 577 /*
578 578 * write CCR4
579 579 */
580 580 movb %dl, %al
581 581 outb $CYRIX_CRD
582 582
583 583 /*
584 584 * select CCR3
585 585 */
586 586 movb $CYRIX_CCR3, %al
587 587 outb $CYRIX_CRI
588 588
589 589 /*
590 590 * disable MAPEN and write CCR3
591 591 */
592 592 movb %cl, %al
593 593 outb $CYRIX_CRD
594 594
595 595 /*
596 596 * restore IMR
597 597 */
598 598 movb %bl, %al
599 599 outb $MIMR_PORT
600 600
601 601 /*
602 602 * test to see if cpuid available
603 603 */
604 604 pushfl
605 605 popl %ecx
606 606 movl %ecx, %eax
607 607 xorl $PS_ID, %eax / try complemented bit
608 608 pushl %eax
609 609 popfl
610 610 pushfl
611 611 popl %eax
612 612 cmpl %eax, %ecx
613 613 jne have_cpuid
614 614 jmp cpu_486
615 615
616 616 restore_PIC:
617 617 /*
618 618 * In case the motherboard/chipset vendor is ignoring line A1 of the
619 619 * I/O address, we set the PIC priorities to sane values.
620 620 */
621 621 movb $0xc7, %al / irq 7 lowest priority
622 622 outb $MCMD_PORT
623 623
624 624 restore_IMR:
625 625 movb %bl, %al
626 626 outb $MIMR_PORT
627 627 jmp cpu_486
628 628
629 629 have_cpuid:
630 630 /*
631 631 * cpuid instruction present
632 632 */
633 633 bts $X86FSET_CPUID, x86_featureset / Just to set; Ignore the CF
634 634 movl $0, %eax
635 635 cpuid
636 636
637 637 movl %ebx, cpu_vendor
638 638 movl %edx, cpu_vendor+4
639 639 movl %ecx, cpu_vendor+8
640 640
641 641 /*
642 642 * early cyrix cpus are somewhat strange and need to be
643 643 * probed in curious ways to determine their identity
644 644 */
645 645
646 646 leal cpu_vendor, %esi
647 647 leal CyrixInstead, %edi
648 648 movl $12, %ecx
649 649 repz
650 650 cmpsb
651 651 je vendor_is_cyrix
652 652
653 653 / let mlsetup()/cpuid_pass1() handle everything else in C
654 654
655 655 jmp cpu_done
656 656
657 657 is486:
658 658 /*
659 659 * test to see if a useful cpuid
660 660 */
661 661 testl %eax, %eax
662 662 jz isa486
663 663
664 664 movl $1, %eax
665 665 cpuid
666 666
667 667 movl %eax, %ebx
668 668 andl $0xF00, %ebx
669 669 cmpl $0x400, %ebx
670 670 je isa486
671 671
672 672 rep; ret /* use 2 byte return instruction */
673 673 /* AMD Software Optimization Guide - Section 6.2 */
674 674 isa486:
675 675 /*
676 676 * lose the return address
677 677 */
678 678 popl %eax
679 679 jmp cpu_486
680 680
681 681 vendor_is_cyrix:
682 682 call is486
683 683
684 684 /*
685 685 * Processor signature and feature flags for Cyrix are insane.
686 686 * BIOS can play with semi-documented registers, so cpuid must be used
687 687 * cautiously. Since we are Cyrix that has cpuid, we have DIR0 and DIR1
688 688 * Keep the family in %ebx and feature flags in %edx until not needed
689 689 */
690 690
691 691 /*
692 692 * read DIR0
693 693 */
694 694 movb $CYRIX_DIR0, %al
695 695 outb $CYRIX_CRI
696 696 inb $CYRIX_CRD
697 697
698 698 /*
699 699 * First we handle the cases where we are a 6x86 or 6x86L.
700 700 * The 6x86 is basically a 486, the only reliable bit in the
701 701 * feature flags is for FPU. The 6x86L is better, unfortunately
702 702 * there is no really good way to distinguish between these two
703 703 * cpu's. We are pessimistic and when in doubt assume 6x86.
704 704 */
705 705
706 706 cmpb $0x40, %al
707 707 jae maybeGX
708 708
709 709 /*
710 710 * We are an M1, either a 6x86 or 6x86L.
711 711 */
712 712 cmpb $0x30, %al
713 713 je maybe6x86L
714 714 cmpb $0x31, %al
715 715 je maybe6x86L
716 716 cmpb $0x34, %al
717 717 je maybe6x86L
718 718 cmpb $0x35, %al
719 719 je maybe6x86L
720 720
721 721 /*
722 722 * although it is possible that we are a 6x86L, the cpu and
723 723 * documentation are so buggy, we just do not care.
724 724 */
725 725 jmp likely6x86
726 726
727 727 maybe6x86L:
728 728 /*
729 729 * read DIR1
730 730 */
731 731 movb $CYRIX_DIR1, %al
732 732 outb $CYRIX_CRI
733 733 inb $CYRIX_CRD
734 734 cmpb $0x22, %al
735 735 jb likely6x86
736 736
737 737 /*
738 738 * We are a 6x86L, or at least a 6x86 with honest cpuid feature flags
739 739 */
740 740 movl $X86_TYPE_CYRIX_6x86L, x86_type
741 741 jmp coma_bug
742 742
743 743 likely6x86:
744 744 /*
745 745 * We are likely a 6x86, or a 6x86L without a way of knowing
746 746 *
747 747 * The 6x86 has NO Pentium or Pentium Pro compatible features even
748 748 * though it claims to be a Pentium Pro compatible!
749 749 *
750 750 * The 6x86 core used in the 6x86 may have most of the Pentium system
751 751 * registers and largely conform to the Pentium System Programming
752 752 * Reference. Documentation on these parts is long gone. Treat it as
753 753 * a crippled Pentium and hope for the best.
754 754 */
755 755
756 756 movl $X86_TYPE_CYRIX_6x86, x86_type
757 757 jmp coma_bug
758 758
759 759 maybeGX:
760 760 /*
761 761 * Now we check whether we are a MediaGX or GXm. We have particular
762 762 * reason for concern here. Even though most of the GXm's
763 763 * report having TSC in the cpuid feature flags, the TSC may be
764 764 * horribly broken. What is worse, is that MediaGX's are basically
765 765 * 486's while the good GXm's are more like Pentium Pro's!
766 766 */
767 767
768 768 cmpb $0x50, %al
769 769 jae maybeM2
770 770
771 771 /*
772 772 * We are either a MediaGX (sometimes called a Gx86) or GXm
773 773 */
774 774
775 775 cmpb $41, %al
776 776 je maybeMediaGX
777 777
778 778 cmpb $44, %al
779 779 jb maybeGXm
780 780
781 781 cmpb $47, %al
782 782 jbe maybeMediaGX
783 783
784 784 /*
785 785 * We do not honestly know what we are, so assume a MediaGX
786 786 */
787 787 jmp media_gx
788 788
789 789 maybeGXm:
790 790 /*
791 791 * It is still possible we are either a MediaGX or GXm, trust cpuid
792 792 * family should be 5 on a GXm
793 793 */
794 794 cmpl $0x500, %ebx
795 795 je GXm
796 796
797 797 /*
798 798 * BIOS/Cyrix might set family to 6 on a GXm
799 799 */
800 800 cmpl $0x600, %ebx
801 801 jne media_gx
802 802
803 803 GXm:
804 804 movl $X86_TYPE_CYRIX_GXm, x86_type
805 805 jmp cpu_done
806 806
807 807 maybeMediaGX:
808 808 /*
809 809 * read DIR1
810 810 */
811 811 movb $CYRIX_DIR1, %al
812 812 outb $CYRIX_CRI
813 813 inb $CYRIX_CRD
814 814
815 815 cmpb $0x30, %al
816 816 jae maybeGXm
817 817
818 818 /*
819 819 * we are a MediaGX for which we do not trust cpuid
820 820 */
821 821 media_gx:
822 822 movl $X86_TYPE_CYRIX_MediaGX, x86_type
823 823 jmp cpu_486
824 824
825 825 maybeM2:
826 826 /*
827 827 * Now we check whether we are a 6x86MX or MII. These cpu's are
828 828 * virtually identical, but we care because for the 6x86MX, we
829 829 * must work around the coma bug. Also for 6x86MX prior to revision
830 830 * 1.4, the TSC may have serious bugs.
831 831 */
832 832
833 833 cmpb $0x60, %al
834 834 jae maybeM3
835 835
836 836 /*
837 837 * family should be 6, but BIOS/Cyrix might set it to 5
838 838 */
839 839 cmpl $0x600, %ebx
840 840 ja cpu_486
841 841
842 842 /*
843 843 * read DIR1
844 844 */
845 845 movb $CYRIX_DIR1, %al
846 846 outb $CYRIX_CRI
847 847 inb $CYRIX_CRD
848 848
849 849 cmpb $0x8, %al
850 850 jb cyrix6x86MX
851 851 cmpb $0x80, %al
852 852 jb MII
853 853
854 854 cyrix6x86MX:
855 855 /*
856 856 * It is altogether unclear how the revision stamped on the cpu
857 857 * maps to the values in DIR0 and DIR1. Just assume TSC is broken.
858 858 */
859 859 movl $X86_TYPE_CYRIX_6x86MX, x86_type
860 860 jmp coma_bug
861 861
862 862 MII:
863 863 movl $X86_TYPE_CYRIX_MII, x86_type
864 864 likeMII:
865 865 jmp cpu_done
866 866
867 867 maybeM3:
868 868 /*
869 869 * We are some chip that we cannot identify yet, an MIII perhaps.
870 870 * We will be optimistic and hope that the chip is much like an MII,
871 871 * and that cpuid is sane. Cyrix seemed to have gotten it right in
872 872 * time for the MII, we can only hope it stayed that way.
873 873 * Maybe the BIOS or Cyrix is trying to hint at something
874 874 */
875 875 cmpl $0x500, %ebx
876 876 je GXm
877 877
878 878 cmpb $0x80, %al
879 879 jae likelyM3
880 880
881 881 /*
882 882 * Just test for the features Cyrix is known for
883 883 */
884 884
885 885 jmp MII
886 886
887 887 likelyM3:
888 888 /*
889 889 * DIR0 with values from 0x80 to 0x8f indicates a VIA Cyrix III, aka
890 890 * the Cyrix MIII. There may be parts later that use the same ranges
891 891 * for DIR0 with special values in DIR1, maybe the VIA CIII, but for
892 892 * now we will call anything with a DIR0 of 0x80 or higher an MIII.
893 893 * The MIII is supposed to support large pages, but we will believe
894 894 * it when we see it. For now we just enable and test for MII features.
895 895 */
896 896 movl $X86_TYPE_VIA_CYRIX_III, x86_type
897 897 jmp likeMII
898 898
899 899 coma_bug:
900 900
901 901 /*
902 902 * With NO_LOCK set to 0 in CCR1, the usual state that BIOS enforces, some
903 903 * bus cycles are issued with LOCK# asserted. With NO_LOCK set to 1, all bus
904 904 * cycles except page table accesses and interrupt ACK cycles do not assert
905 905 * LOCK#. xchgl is an instruction that asserts LOCK# if NO_LOCK is set to 0.
906 906 * Due to a bug in the cpu core involving over-optimization of branch
907 907 * prediction, register renaming, and execution of instructions down both the
908 908 * X and Y pipes for the xchgl instruction, short loops can be written that
909 909 * never de-assert LOCK# from one invocation of the loop to the next, ad
910 910 * infinitum. The undesirable effect of this situation is that interrupts are
911 911 * not serviced. The ideal workaround to this bug would be to set NO_LOCK to
912 912 * 1. Unfortunately bus cycles that would otherwise have asserted LOCK# no
913 913 * longer do, unless they are page table accesses or interrupt ACK cycles.
914 914 * With LOCK# not asserted, these bus cycles are now cached. This can cause
915 915 * undesirable behaviour if the ARR's are not configured correctly. Solaris
916 916 * does not configure the ARR's, nor does it provide any useful mechanism for
917 917 * doing so, thus the ideal workaround is not viable. Fortunately, the only
918 918 * known exploits for this bug involve the xchgl instruction specifically.
919 919 * There is a group of undocumented registers on Cyrix 6x86, 6x86L, and
920 920 * 6x86MX cpu's which can be used to specify one instruction as a serializing
921 921 * instruction. With the xchgl instruction serialized, LOCK# is still
922 922 * asserted, but it is the sole instruction for which LOCK# is asserted.
923 923 * There is now some added penalty for the xchgl instruction, but the usual
924 924 * bus locking is preserved. This ingenious workaround was discovered by
925 925 * disassembling a binary provided by Cyrix as a workaround for this bug on
926 926 * Windows, but its not documented anywhere by Cyrix, nor is the bug actually
927 927 * mentioned in any public errata! The only concern for this workaround is
928 928 * that there may be similar undiscovered bugs with other instructions that
929 929 * assert LOCK# that may be leveraged to similar ends. The fact that Cyrix
930 930 * fixed this bug sometime late in 1997 and no other exploits other than
931 931 * xchgl have been discovered is good indication that this workaround is
932 932 * reasonable.
933 933 */
934 934
935 935 .set CYRIX_DBR0, 0x30 / Debug Register 0
936 936 .set CYRIX_DBR1, 0x31 / Debug Register 1
937 937 .set CYRIX_DBR2, 0x32 / Debug Register 2
938 938 .set CYRIX_DBR3, 0x33 / Debug Register 3
939 939 .set CYRIX_DOR, 0x3c / Debug Opcode Register
940 940
941 941 /*
942 942 * What is known about DBR1, DBR2, DBR3, and DOR is that for normal
943 943 * cpu execution DBR1, DBR2, and DBR3 are set to 0. To obtain opcode
944 944 * serialization, DBR1, DBR2, and DBR3 are loaded with 0xb8, 0x7f,
945 945 * and 0xff. Then, DOR is loaded with the one byte opcode.
946 946 */
947 947
948 948 /*
949 949 * select CCR3
950 950 */
951 951 movb $CYRIX_CCR3, %al
952 952 outb $CYRIX_CRI
953 953
954 954 /*
955 955 * read CCR3 and mask out MAPEN
956 956 */
957 957 inb $CYRIX_CRD
958 958 andb $0xf, %al
959 959
960 960 /*
961 961 * save masked CCR3 in %ah
962 962 */
963 963 movb %al, %ah
964 964
965 965 /*
966 966 * select CCR3
967 967 */
968 968 movb $CYRIX_CCR3, %al
969 969 outb $CYRIX_CRI
970 970
971 971 /*
972 972 * enable MAPEN
973 973 */
974 974 movb %ah, %al
975 975 orb $0x10, %al
976 976 outb $CYRIX_CRD
977 977
978 978 /*
979 979 * read DBR0
980 980 */
981 981 movb $CYRIX_DBR0, %al
982 982 outb $CYRIX_CRI
983 983 inb $CYRIX_CRD
984 984
985 985 /*
986 986 * disable MATCH and save in %bh
987 987 */
988 988 orb $0x80, %al
989 989 movb %al, %bh
990 990
991 991 /*
992 992 * write DBR0
993 993 */
994 994 movb $CYRIX_DBR0, %al
995 995 outb $CYRIX_CRI
996 996 movb %bh, %al
997 997 outb $CYRIX_CRD
998 998
999 999 /*
1000 1000 * write DBR1
1001 1001 */
1002 1002 movb $CYRIX_DBR1, %al
1003 1003 outb $CYRIX_CRI
1004 1004 movb $0xf8, %al
1005 1005 outb $CYRIX_CRD
1006 1006
1007 1007 /*
1008 1008 * write DBR2
1009 1009 */
1010 1010 movb $CYRIX_DBR2, %al
1011 1011 outb $CYRIX_CRI
1012 1012 movb $0x7f, %al
1013 1013 outb $CYRIX_CRD
1014 1014
1015 1015 /*
1016 1016 * write DBR3
1017 1017 */
1018 1018 movb $CYRIX_DBR3, %al
1019 1019 outb $CYRIX_CRI
1020 1020 xorb %al, %al
1021 1021 outb $CYRIX_CRD
1022 1022
1023 1023 /*
1024 1024 * write DOR
1025 1025 */
1026 1026 movb $CYRIX_DOR, %al
1027 1027 outb $CYRIX_CRI
1028 1028 movb $0x87, %al
1029 1029 outb $CYRIX_CRD
1030 1030
1031 1031 /*
1032 1032 * enable MATCH
1033 1033 */
1034 1034 movb $CYRIX_DBR0, %al
1035 1035 outb $CYRIX_CRI
1036 1036 movb %bh, %al
1037 1037 andb $0x7f, %al
1038 1038 outb $CYRIX_CRD
1039 1039
1040 1040 /*
1041 1041 * disable MAPEN
1042 1042 */
1043 1043 movb $0xc3, %al
1044 1044 outb $CYRIX_CRI
1045 1045 movb %ah, %al
1046 1046 outb $CYRIX_CRD
1047 1047
1048 1048 jmp cpu_done
1049 1049
1050 1050 cpu_done:
1051 1051
1052 1052 popfl /* Restore original FLAGS */
1053 1053 popal /* Restore all registers */
1054 1054
1055 1055 #endif /* !__xpv */
1056 1056
1057 1057 /*
1058 1058 * mlsetup(%esp) gets called.
1059 1059 */
1060 1060 pushl %esp
1061 1061 call mlsetup
1062 1062 addl $4, %esp
1063 1063
1064 1064 /*
1065 1065 * We change our appearance to look like the real thread 0.
1066 1066 * (NOTE: making ourselves to be a real thread may be a noop)
1067 1067 * main() gets called. (NOTE: main() never returns).
1068 1068 */
1069 1069 call main
1070 1070 /* NOTREACHED */
1071 1071 pushl $__return_from_main
1072 1072 call panic
1073 1073
1074 1074 /* NOTREACHED */
1075 1075 cpu_486:
1076 1076 pushl $__unsupported_cpu
1077 1077 call panic
1078 1078 SET_SIZE(_locore_start)
1079 1079
1080 1080 #endif /* __lint */
1081 1081 #endif /* !__amd64 */
1082 1082
1083 1083
1084 1084 /*
1085 1085 * For stack layout, see privregs.h
1086 1086 * When cmntrap gets called, the error code and trap number have been pushed.
1087 1087 * When cmntrap_pushed gets called, the entire struct regs has been pushed.
1088 1088 */
1089 1089
1090 1090 #if defined(__lint)
1091 1091
1092 1092 /* ARGSUSED */
1093 1093 void
1094 1094 cmntrap()
1095 1095 {}
1096 1096
1097 1097 #else /* __lint */
1098 1098
1099 1099 .globl trap /* C handler called below */
1100 1100
1101 1101 #if defined(__amd64)
1102 1102
1103 1103 ENTRY_NP2(cmntrap, _cmntrap)
1104 1104
1105 1105 INTR_PUSH
1106 1106
1107 1107 ALTENTRY(cmntrap_pushed)
1108 1108
1109 1109 movq %rsp, %rbp
1110 1110
1111 1111 /*
1112 1112 * - if this is a #pf i.e. T_PGFLT, %r15 is live
1113 1113 * and contains the faulting address i.e. a copy of %cr2
1114 1114 *
1115 1115 * - if this is a #db i.e. T_SGLSTP, %r15 is live
1116 1116 * and contains the value of %db6
1117 1117 */
1118 1118
1119 1119 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1120 1120 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1121 1121 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1122 1122
1123 1123 /*
1124 1124 * We must first check if DTrace has set its NOFAULT bit. This
1125 1125 * regrettably must happen before the trap stack is recorded, because
1126 1126 * this requires a call to getpcstack() and may induce recursion if an
1127 1127 * fbt::getpcstack: enabling is inducing the bad load.
1128 1128 */
1129 1129 movl %gs:CPU_ID, %eax
1130 1130 shlq $CPU_CORE_SHIFT, %rax
1131 1131 leaq cpu_core(%rip), %r8
1132 1132 addq %r8, %rax
1133 1133 movw CPUC_DTRACE_FLAGS(%rax), %cx
1134 1134 testw $CPU_DTRACE_NOFAULT, %cx
1135 1135 jnz .dtrace_induced
1136 1136
1137 1137 TRACE_STACK(%rdi)
1138 1138
1139 1139 movq %rbp, %rdi
1140 1140 movq %r15, %rsi
1141 1141 movl %gs:CPU_ID, %edx
1142 1142
1143 1143 /*
1144 1144 * We know that this isn't a DTrace non-faulting load; we can now safely
1145 1145 * reenable interrupts. (In the case of pagefaults, we enter through an
1146 1146 * interrupt gate.)
1147 1147 */
1148 1148 ENABLE_INTR_FLAGS
1149 1149
1150 1150 call trap /* trap(rp, addr, cpuid) handles all traps */
1151 1151 jmp _sys_rtt
1152 1152
1153 1153 .dtrace_induced:
1154 1154 cmpw $KCS_SEL, REGOFF_CS(%rbp) /* test CS for user-mode trap */
1155 1155 jne 3f /* if from user, panic */
1156 1156
1157 1157 cmpl $T_PGFLT, REGOFF_TRAPNO(%rbp)
1158 1158 je 1f
1159 1159
1160 1160 cmpl $T_GPFLT, REGOFF_TRAPNO(%rbp)
1161 1161 je 0f
1162 1162
1163 1163 cmpl $T_ILLINST, REGOFF_TRAPNO(%rbp)
1164 1164 je 0f
1165 1165
1166 1166 cmpl $T_ZERODIV, REGOFF_TRAPNO(%rbp)
1167 1167 jne 4f /* if not PF/GP/UD/DE, panic */
1168 1168
1169 1169 orw $CPU_DTRACE_DIVZERO, %cx
1170 1170 movw %cx, CPUC_DTRACE_FLAGS(%rax)
1171 1171 jmp 2f
1172 1172
1173 1173 /*
1174 1174 * If we've taken a GPF, we don't (unfortunately) have the address that
1175 1175 * induced the fault. So instead of setting the fault to BADADDR,
1176 1176 * we'll set the fault to ILLOP.
1177 1177 */
1178 1178 0:
1179 1179 orw $CPU_DTRACE_ILLOP, %cx
1180 1180 movw %cx, CPUC_DTRACE_FLAGS(%rax)
1181 1181 jmp 2f
1182 1182 1:
1183 1183 orw $CPU_DTRACE_BADADDR, %cx
|
↓ open down ↓ |
1183 lines elided |
↑ open up ↑ |
1184 1184 movw %cx, CPUC_DTRACE_FLAGS(%rax) /* set fault to bad addr */
1185 1185 movq %r15, CPUC_DTRACE_ILLVAL(%rax)
1186 1186 /* fault addr is illegal value */
1187 1187 2:
1188 1188 movq REGOFF_RIP(%rbp), %rdi
1189 1189 movq %rdi, %r12
1190 1190 call dtrace_instr_size
1191 1191 addq %rax, %r12
1192 1192 movq %r12, REGOFF_RIP(%rbp)
1193 1193 INTR_POP
1194 - call *x86_md_clear
1194 + call x86_md_clear
1195 1195 jmp tr_iret_auto
1196 1196 /*NOTREACHED*/
1197 1197 3:
1198 1198 leaq dtrace_badflags(%rip), %rdi
1199 1199 xorl %eax, %eax
1200 1200 call panic
1201 1201 4:
1202 1202 leaq dtrace_badtrap(%rip), %rdi
1203 1203 xorl %eax, %eax
1204 1204 call panic
1205 1205 SET_SIZE(cmntrap_pushed)
1206 1206 SET_SIZE(cmntrap)
1207 1207 SET_SIZE(_cmntrap)
1208 1208
1209 1209 #elif defined(__i386)
1210 1210
1211 1211
1212 1212 ENTRY_NP2(cmntrap, _cmntrap)
1213 1213
1214 1214 INTR_PUSH
1215 1215
1216 1216 ALTENTRY(cmntrap_pushed)
1217 1217
1218 1218 movl %esp, %ebp
1219 1219
1220 1220 /*
1221 1221 * - if this is a #pf i.e. T_PGFLT, %esi is live
1222 1222 * and contains the faulting address i.e. a copy of %cr2
1223 1223 *
1224 1224 * - if this is a #db i.e. T_SGLSTP, %esi is live
1225 1225 * and contains the value of %db6
1226 1226 */
1227 1227
1228 1228 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1229 1229 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1230 1230 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1231 1231
1232 1232 /*
1233 1233 * We must first check if DTrace has set its NOFAULT bit. This
1234 1234 * regrettably must happen before the trap stack is recorded, because
1235 1235 * this requires a call to getpcstack() and may induce recursion if an
1236 1236 * fbt::getpcstack: enabling is inducing the bad load.
1237 1237 */
1238 1238 movl %gs:CPU_ID, %eax
1239 1239 shll $CPU_CORE_SHIFT, %eax
1240 1240 addl $cpu_core, %eax
1241 1241 movw CPUC_DTRACE_FLAGS(%eax), %cx
1242 1242 testw $CPU_DTRACE_NOFAULT, %cx
1243 1243 jnz .dtrace_induced
1244 1244
1245 1245 TRACE_STACK(%edi)
1246 1246
1247 1247 pushl %gs:CPU_ID
1248 1248 pushl %esi /* fault address for PGFLTs */
1249 1249 pushl %ebp /* ®s */
1250 1250
1251 1251 /*
1252 1252 * We know that this isn't a DTrace non-faulting load; we can now safely
1253 1253 * reenable interrupts. (In the case of pagefaults, we enter through an
1254 1254 * interrupt gate.)
1255 1255 */
1256 1256 ENABLE_INTR_FLAGS
1257 1257
1258 1258 call trap /* trap(rp, addr, cpuid) handles all traps */
1259 1259 addl $12, %esp /* get argument off stack */
1260 1260 jmp _sys_rtt
1261 1261
1262 1262 .dtrace_induced:
1263 1263 cmpw $KCS_SEL, REGOFF_CS(%ebp) /* test CS for user-mode trap */
1264 1264 jne 3f /* if from user, panic */
1265 1265
1266 1266 cmpl $T_PGFLT, REGOFF_TRAPNO(%ebp)
1267 1267 je 1f
1268 1268
1269 1269 cmpl $T_GPFLT, REGOFF_TRAPNO(%ebp)
1270 1270 je 0f
1271 1271
1272 1272 cmpl $T_ZERODIV, REGOFF_TRAPNO(%ebp)
1273 1273 jne 4f /* if not PF/GP/UD/DE, panic */
1274 1274
1275 1275 orw $CPU_DTRACE_DIVZERO, %cx
1276 1276 movw %cx, CPUC_DTRACE_FLAGS(%eax)
1277 1277 jmp 2f
1278 1278
1279 1279 0:
1280 1280 /*
1281 1281 * If we've taken a GPF, we don't (unfortunately) have the address that
1282 1282 * induced the fault. So instead of setting the fault to BADADDR,
1283 1283 * we'll set the fault to ILLOP.
1284 1284 */
1285 1285 orw $CPU_DTRACE_ILLOP, %cx
1286 1286 movw %cx, CPUC_DTRACE_FLAGS(%eax)
1287 1287 jmp 2f
1288 1288 1:
1289 1289 orw $CPU_DTRACE_BADADDR, %cx
1290 1290 movw %cx, CPUC_DTRACE_FLAGS(%eax) /* set fault to bad addr */
1291 1291 movl %esi, CPUC_DTRACE_ILLVAL(%eax)
1292 1292 /* fault addr is illegal value */
1293 1293 2:
1294 1294 pushl REGOFF_EIP(%ebp)
1295 1295 call dtrace_instr_size
1296 1296 addl $4, %esp
1297 1297 movl REGOFF_EIP(%ebp), %ecx
1298 1298 addl %eax, %ecx
1299 1299 movl %ecx, REGOFF_EIP(%ebp)
1300 1300 INTR_POP_KERNEL
1301 1301 IRET
1302 1302 /*NOTREACHED*/
1303 1303 3:
1304 1304 pushl $dtrace_badflags
1305 1305 call panic
1306 1306 4:
1307 1307 pushl $dtrace_badtrap
1308 1308 call panic
1309 1309 SET_SIZE(cmntrap)
1310 1310 SET_SIZE(_cmntrap)
1311 1311
1312 1312 #endif /* __i386 */
1313 1313
1314 1314 /*
1315 1315 * Declare a uintptr_t which has the size of _cmntrap to enable stack
1316 1316 * traceback code to know when a regs structure is on the stack.
1317 1317 */
1318 1318 .globl _cmntrap_size
1319 1319 .align CLONGSIZE
1320 1320 _cmntrap_size:
1321 1321 .NWORD . - _cmntrap
1322 1322 .type _cmntrap_size, @object
1323 1323
1324 1324 dtrace_badflags:
1325 1325 .string "bad DTrace flags"
1326 1326
1327 1327 dtrace_badtrap:
1328 1328 .string "bad DTrace trap"
1329 1329
1330 1330 #endif /* __lint */
1331 1331
1332 1332 #if defined(__lint)
1333 1333
1334 1334 /* ARGSUSED */
1335 1335 void
1336 1336 cmninttrap()
1337 1337 {}
1338 1338
1339 1339 #if !defined(__xpv)
1340 1340 void
1341 1341 bop_trap_handler(void)
1342 1342 {}
1343 1343 #endif
1344 1344
1345 1345 #else /* __lint */
1346 1346
1347 1347 .globl trap /* C handler called below */
1348 1348
1349 1349 #if defined(__amd64)
1350 1350
1351 1351 ENTRY_NP(cmninttrap)
1352 1352
1353 1353 INTR_PUSH
1354 1354 INTGATE_INIT_KERNEL_FLAGS
1355 1355
1356 1356 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1357 1357 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1358 1358 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1359 1359
1360 1360 movq %rsp, %rbp
1361 1361
1362 1362 movl %gs:CPU_ID, %edx
1363 1363 xorl %esi, %esi
1364 1364 movq %rsp, %rdi
1365 1365 call trap /* trap(rp, addr, cpuid) handles all traps */
1366 1366 jmp _sys_rtt
1367 1367 SET_SIZE(cmninttrap)
1368 1368
1369 1369 #if !defined(__xpv)
1370 1370 /*
1371 1371 * Handle traps early in boot. Just revectors into C quickly as
1372 1372 * these are always fatal errors.
1373 1373 *
1374 1374 * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap().
1375 1375 */
1376 1376 ENTRY(bop_trap_handler)
1377 1377 movq %rsp, %rdi
1378 1378 sub $8, %rsp
1379 1379 call bop_trap
1380 1380 SET_SIZE(bop_trap_handler)
1381 1381 #endif
1382 1382
1383 1383 #elif defined(__i386)
1384 1384
1385 1385 ENTRY_NP(cmninttrap)
1386 1386
1387 1387 INTR_PUSH
1388 1388 INTGATE_INIT_KERNEL_FLAGS
1389 1389
1390 1390 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1391 1391 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1392 1392 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1393 1393
1394 1394 movl %esp, %ebp
1395 1395
1396 1396 TRACE_STACK(%edi)
1397 1397
1398 1398 pushl %gs:CPU_ID
1399 1399 pushl $0
1400 1400 pushl %ebp
1401 1401 call trap /* trap(rp, addr, cpuid) handles all traps */
1402 1402 addl $12, %esp
1403 1403 jmp _sys_rtt
1404 1404 SET_SIZE(cmninttrap)
1405 1405
1406 1406 #if !defined(__xpv)
1407 1407 /*
1408 1408 * Handle traps early in boot. Just revectors into C quickly as
1409 1409 * these are always fatal errors.
1410 1410 */
1411 1411 ENTRY(bop_trap_handler)
1412 1412 movl %esp, %eax
1413 1413 pushl %eax
1414 1414 call bop_trap
1415 1415 SET_SIZE(bop_trap_handler)
1416 1416 #endif
1417 1417
1418 1418 #endif /* __i386 */
1419 1419
1420 1420 #endif /* __lint */
1421 1421
1422 1422 #if defined(__lint)
1423 1423
1424 1424 /* ARGSUSED */
1425 1425 void
1426 1426 dtrace_trap()
1427 1427 {}
1428 1428
1429 1429 #else /* __lint */
1430 1430
1431 1431 .globl dtrace_user_probe
1432 1432
1433 1433 #if defined(__amd64)
1434 1434
1435 1435 ENTRY_NP(dtrace_trap)
1436 1436
1437 1437 INTR_PUSH
1438 1438
1439 1439 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1440 1440 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1441 1441 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1442 1442
1443 1443 movq %rsp, %rbp
1444 1444
1445 1445 movl %gs:CPU_ID, %edx
1446 1446 #if defined(__xpv)
1447 1447 movq %gs:CPU_VCPU_INFO, %rsi
1448 1448 movq VCPU_INFO_ARCH_CR2(%rsi), %rsi
1449 1449 #else
1450 1450 movq %cr2, %rsi
1451 1451 #endif
1452 1452 movq %rsp, %rdi
1453 1453
1454 1454 ENABLE_INTR_FLAGS
1455 1455
1456 1456 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
1457 1457 jmp _sys_rtt
1458 1458
1459 1459 SET_SIZE(dtrace_trap)
1460 1460
1461 1461 #elif defined(__i386)
1462 1462
1463 1463 ENTRY_NP(dtrace_trap)
1464 1464
1465 1465 INTR_PUSH
1466 1466
1467 1467 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1468 1468 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1469 1469 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1470 1470
1471 1471 movl %esp, %ebp
1472 1472
1473 1473 pushl %gs:CPU_ID
1474 1474 #if defined(__xpv)
1475 1475 movl %gs:CPU_VCPU_INFO, %eax
1476 1476 movl VCPU_INFO_ARCH_CR2(%eax), %eax
1477 1477 #else
1478 1478 movl %cr2, %eax
1479 1479 #endif
1480 1480 pushl %eax
1481 1481 pushl %ebp
1482 1482
1483 1483 ENABLE_INTR_FLAGS
1484 1484
1485 1485 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
1486 1486 addl $12, %esp /* get argument off stack */
1487 1487
1488 1488 jmp _sys_rtt
1489 1489 SET_SIZE(dtrace_trap)
1490 1490
1491 1491 #endif /* __i386 */
1492 1492
1493 1493 #endif /* __lint */
1494 1494
1495 1495 /*
1496 1496 * Return from _sys_trap routine.
1497 1497 */
1498 1498
1499 1499 #if defined(__lint)
1500 1500
1501 1501 void
1502 1502 lwp_rtt_initial(void)
1503 1503 {}
1504 1504
1505 1505 void
1506 1506 lwp_rtt(void)
1507 1507 {}
1508 1508
1509 1509 void
1510 1510 _sys_rtt(void)
1511 1511 {}
1512 1512
1513 1513 #else /* __lint */
1514 1514
1515 1515 ENTRY_NP(lwp_rtt_initial)
1516 1516 movq %gs:CPU_THREAD, %r15
1517 1517 movq T_STACK(%r15), %rsp /* switch to the thread stack */
1518 1518 movq %rsp, %rbp
1519 1519 call __dtrace_probe___proc_start
1520 1520 jmp _lwp_rtt
1521 1521
1522 1522 ENTRY_NP(lwp_rtt)
1523 1523
1524 1524 /*
1525 1525 * r14 lwp
1526 1526 * rdx lwp->lwp_procp
1527 1527 * r15 curthread
1528 1528 */
1529 1529
1530 1530 movq %gs:CPU_THREAD, %r15
1531 1531 movq T_STACK(%r15), %rsp /* switch to the thread stack */
1532 1532 movq %rsp, %rbp
1533 1533 _lwp_rtt:
1534 1534 call __dtrace_probe___proc_lwp__start
1535 1535 movq %gs:CPU_LWP, %r14
1536 1536 movq LWP_PROCP(%r14), %rdx
1537 1537
1538 1538 /*
1539 1539 * XX64 Is the stack misaligned correctly at this point?
1540 1540 * If not, we need to do a push before calling anything ..
1541 1541 */
1542 1542
1543 1543 #if defined(DEBUG)
1544 1544 /*
1545 1545 * If we were to run lwp_savectx at this point -without-
1546 1546 * pcb_rupdate being set to 1, we'd end up sampling the hardware
1547 1547 * state left by the previous running lwp, rather than setting
1548 1548 * the values requested by the lwp creator. Bad.
1549 1549 */
1550 1550 testb $0x1, PCB_RUPDATE(%r14)
1551 1551 jne 1f
1552 1552 leaq _no_pending_updates(%rip), %rdi
1553 1553 movl $__LINE__, %esi
1554 1554 movq %r14, %rdx
1555 1555 xorl %eax, %eax
1556 1556 call panic
1557 1557 1:
1558 1558 #endif
1559 1559
1560 1560 /*
1561 1561 * If agent lwp, clear %fs and %gs
1562 1562 */
1563 1563 cmpq %r15, P_AGENTTP(%rdx)
1564 1564 jne 1f
1565 1565 xorl %ecx, %ecx
1566 1566 movq %rcx, REGOFF_FS(%rsp)
1567 1567 movq %rcx, REGOFF_GS(%rsp)
1568 1568 movw %cx, LWP_PCB_FS(%r14)
1569 1569 movw %cx, LWP_PCB_GS(%r14)
1570 1570 1:
1571 1571 call dtrace_systrace_rtt
1572 1572 movq REGOFF_RDX(%rsp), %rsi
1573 1573 movq REGOFF_RAX(%rsp), %rdi
1574 1574 call post_syscall /* post_syscall(rval1, rval2) */
1575 1575
1576 1576 /*
1577 1577 * XXX - may want a fast path that avoids sys_rtt_common in the
1578 1578 * most common case.
1579 1579 */
1580 1580 ALTENTRY(_sys_rtt)
1581 1581 CLI(%rax) /* disable interrupts */
1582 1582 ALTENTRY(_sys_rtt_ints_disabled)
1583 1583 movq %rsp, %rdi /* pass rp to sys_rtt_common */
1584 1584 call sys_rtt_common /* do common sys_rtt tasks */
1585 1585 testq %rax, %rax /* returning to userland? */
1586 1586 jz sr_sup
1587 1587
1588 1588 /*
1589 1589 * Return to user
|
↓ open down ↓ |
385 lines elided |
↑ open up ↑ |
1590 1590 */
1591 1591 ASSERT_UPCALL_MASK_IS_SET
1592 1592 cmpw $UCS_SEL, REGOFF_CS(%rsp) /* test for native (64-bit) lwp? */
1593 1593 je sys_rtt_syscall
1594 1594
1595 1595 /*
1596 1596 * Return to 32-bit userland
1597 1597 */
1598 1598 ALTENTRY(sys_rtt_syscall32)
1599 1599 USER32_POP
1600 - call *x86_md_clear
1600 + call x86_md_clear
1601 1601 jmp tr_iret_user
1602 1602 /*NOTREACHED*/
1603 1603
1604 1604 ALTENTRY(sys_rtt_syscall)
1605 1605 /*
1606 1606 * Return to 64-bit userland
1607 1607 */
1608 1608 USER_POP
1609 1609 ALTENTRY(nopop_sys_rtt_syscall)
1610 - call *x86_md_clear
1610 + call x86_md_clear
1611 1611 jmp tr_iret_user
1612 1612 /*NOTREACHED*/
1613 1613 SET_SIZE(nopop_sys_rtt_syscall)
1614 1614
1615 1615 /*
1616 1616 * Return to supervisor
1617 1617 * NOTE: to make the check in trap() that tests if we are executing
1618 1618 * segment register fixup/restore code work properly, sr_sup MUST be
1619 1619 * after _sys_rtt .
1620 1620 */
1621 1621 ALTENTRY(sr_sup)
1622 1622 /*
1623 1623 * Restore regs before doing iretq to kernel mode
1624 1624 */
1625 1625 INTR_POP
1626 1626 jmp tr_iret_kernel
1627 1627 .globl _sys_rtt_end
1628 1628 _sys_rtt_end:
1629 1629 /*NOTREACHED*/
1630 1630 SET_SIZE(sr_sup)
1631 1631 SET_SIZE(_sys_rtt_end)
1632 1632 SET_SIZE(lwp_rtt)
1633 1633 SET_SIZE(lwp_rtt_initial)
1634 1634 SET_SIZE(_sys_rtt_ints_disabled)
1635 1635 SET_SIZE(_sys_rtt)
1636 1636 SET_SIZE(sys_rtt_syscall)
1637 1637 SET_SIZE(sys_rtt_syscall32)
1638 1638
1639 1639 #endif /* __lint */
1640 1640
1641 1641 #if defined(__lint)
1642 1642
1643 1643 /*
1644 1644 * So why do we have to deal with all this crud in the world of ia32?
1645 1645 *
1646 1646 * Basically there are four classes of ia32 implementations, those that do not
1647 1647 * have a TSC, those that have a marginal TSC that is broken to the extent
1648 1648 * that it is useless, those that have a marginal TSC that is not quite so
1649 1649 * horribly broken and can be used with some care, and those that have a
1650 1650 * reliable TSC. This crud has to be here in order to sift through all the
1651 1651 * variants.
1652 1652 */
1653 1653
1654 1654 /*ARGSUSED*/
1655 1655 uint64_t
1656 1656 freq_tsc(uint32_t *pit_counter)
1657 1657 {
1658 1658 return (0);
1659 1659 }
1660 1660
1661 1661 #else /* __lint */
1662 1662
1663 1663 #if defined(__amd64)
1664 1664
1665 1665 /*
1666 1666 * XX64 quick and dirty port from the i386 version. Since we
1667 1667 * believe the amd64 tsc is more reliable, could this code be
1668 1668 * simpler?
1669 1669 */
1670 1670 ENTRY_NP(freq_tsc)
1671 1671 pushq %rbp
1672 1672 movq %rsp, %rbp
1673 1673 movq %rdi, %r9 /* save pit_counter */
1674 1674 pushq %rbx
1675 1675
1676 1676 / We have a TSC, but we have no way in general to know how reliable it is.
1677 1677 / Usually a marginal TSC behaves appropriately unless not enough time
1678 1678 / elapses between reads. A reliable TSC can be read as often and as rapidly
1679 1679 / as desired. The simplistic approach of reading the TSC counter and
1680 1680 / correlating to the PIT counter cannot be naively followed. Instead estimates
1681 1681 / have to be taken to successively refine a guess at the speed of the cpu
1682 1682 / and then the TSC and PIT counter are correlated. In practice very rarely
1683 1683 / is more than one quick loop required for an estimate. Measures have to be
1684 1684 / taken to prevent the PIT counter from wrapping beyond its resolution and for
1685 1685 / measuring the clock rate of very fast processors.
1686 1686 /
1687 1687 / The following constant can be tuned. It should be such that the loop does
1688 1688 / not take too many nor too few PIT counts to execute. If this value is too
1689 1689 / large, then on slow machines the loop will take a long time, or the PIT
1690 1690 / counter may even wrap. If this value is too small, then on fast machines
1691 1691 / the PIT counter may count so few ticks that the resolution of the PIT
1692 1692 / itself causes a bad guess. Because this code is used in machines with
1693 1693 / marginal TSC's and/or IO, if this value is too small on those, it may
1694 1694 / cause the calculated cpu frequency to vary slightly from boot to boot.
1695 1695 /
1696 1696 / In all cases even if this constant is set inappropriately, the algorithm
1697 1697 / will still work and the caller should be able to handle variances in the
1698 1698 / calculation of cpu frequency, but the calculation will be inefficient and
1699 1699 / take a disproportionate amount of time relative to a well selected value.
1700 1700 / As the slowest supported cpu becomes faster, this constant should be
1701 1701 / carefully increased.
1702 1702
1703 1703 movl $0x8000, %ecx
1704 1704
1705 1705 / to make sure the instruction cache has been warmed
1706 1706 clc
1707 1707
1708 1708 jmp freq_tsc_loop
1709 1709
1710 1710 / The following block of code up to and including the latching of the PIT
1711 1711 / counter after freq_tsc_perf_loop is very critical and very carefully
1712 1712 / written, it should only be modified with great care. freq_tsc_loop to
1713 1713 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
1714 1714 / freq_tsc_perf_loop up to the unlatching of the PIT counter.
1715 1715
1716 1716 .align 32
1717 1717 freq_tsc_loop:
1718 1718 / save the loop count in %ebx
1719 1719 movl %ecx, %ebx
1720 1720
1721 1721 / initialize the PIT counter and start a count down
1722 1722 movb $PIT_LOADMODE, %al
1723 1723 outb $PITCTL_PORT
1724 1724 movb $0xff, %al
1725 1725 outb $PITCTR0_PORT
1726 1726 outb $PITCTR0_PORT
1727 1727
1728 1728 / read the TSC and store the TS in %edi:%esi
1729 1729 rdtsc
1730 1730 movl %eax, %esi
1731 1731
1732 1732 freq_tsc_perf_loop:
1733 1733 movl %edx, %edi
1734 1734 movl %eax, %esi
1735 1735 movl %edx, %edi
1736 1736 loop freq_tsc_perf_loop
1737 1737
1738 1738 / read the TSC and store the LSW in %ecx
1739 1739 rdtsc
1740 1740 movl %eax, %ecx
1741 1741
1742 1742 / latch the PIT counter and status
1743 1743 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
1744 1744 outb $PITCTL_PORT
1745 1745
1746 1746 / remember if the icache has been warmed
1747 1747 setc %ah
1748 1748
1749 1749 / read the PIT status
1750 1750 inb $PITCTR0_PORT
1751 1751 shll $8, %eax
1752 1752
1753 1753 / read PIT count
1754 1754 inb $PITCTR0_PORT
1755 1755 shll $8, %eax
1756 1756 inb $PITCTR0_PORT
1757 1757 bswap %eax
1758 1758
1759 1759 / check to see if the PIT count was loaded into the CE
1760 1760 btw $_CONST(PITSTAT_NULLCNT+8), %ax
1761 1761 jc freq_tsc_increase_count
1762 1762
1763 1763 / check to see if PIT counter wrapped
1764 1764 btw $_CONST(PITSTAT_OUTPUT+8), %ax
1765 1765 jnc freq_tsc_pit_did_not_wrap
1766 1766
1767 1767 / halve count
1768 1768 shrl $1, %ebx
1769 1769 movl %ebx, %ecx
1770 1770
1771 1771 / the instruction cache has been warmed
1772 1772 stc
1773 1773
1774 1774 jmp freq_tsc_loop
1775 1775
1776 1776 freq_tsc_increase_count:
1777 1777 shll $1, %ebx
1778 1778 jc freq_tsc_too_fast
1779 1779
1780 1780 movl %ebx, %ecx
1781 1781
1782 1782 / the instruction cache has been warmed
1783 1783 stc
1784 1784
1785 1785 jmp freq_tsc_loop
1786 1786
1787 1787 freq_tsc_pit_did_not_wrap:
1788 1788 roll $16, %eax
1789 1789
1790 1790 cmpw $0x2000, %ax
1791 1791 notw %ax
1792 1792 jb freq_tsc_sufficient_duration
1793 1793
1794 1794 freq_tsc_calculate:
1795 1795 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
1796 1796 / then on the second CLK pulse the CE is decremented, therefore mode 0
1797 1797 / is really a (count + 1) counter, ugh
1798 1798 xorl %esi, %esi
1799 1799 movw %ax, %si
1800 1800 incl %esi
1801 1801
1802 1802 movl $0xf000, %eax
1803 1803 mull %ebx
1804 1804
1805 1805 / tuck away (target_pit_count * loop_count)
1806 1806 movl %edx, %ecx
1807 1807 movl %eax, %ebx
1808 1808
1809 1809 movl %esi, %eax
1810 1810 movl $0xffffffff, %edx
1811 1811 mull %edx
1812 1812
1813 1813 addl %esi, %eax
1814 1814 adcl $0, %edx
1815 1815
1816 1816 cmpl %ecx, %edx
1817 1817 ja freq_tsc_div_safe
1818 1818 jb freq_tsc_too_fast
1819 1819
1820 1820 cmpl %ebx, %eax
1821 1821 jbe freq_tsc_too_fast
1822 1822
1823 1823 freq_tsc_div_safe:
1824 1824 movl %ecx, %edx
1825 1825 movl %ebx, %eax
1826 1826
1827 1827 movl %esi, %ecx
1828 1828 divl %ecx
1829 1829
1830 1830 movl %eax, %ecx
1831 1831
1832 1832 / the instruction cache has been warmed
1833 1833 stc
1834 1834
1835 1835 jmp freq_tsc_loop
1836 1836
1837 1837 freq_tsc_sufficient_duration:
1838 1838 / test to see if the icache has been warmed
1839 1839 btl $16, %eax
1840 1840 jnc freq_tsc_calculate
1841 1841
1842 1842 / recall mode 0 is a (count + 1) counter
1843 1843 andl $0xffff, %eax
1844 1844 incl %eax
1845 1845
1846 1846 / save the number of PIT counts
1847 1847 movl %eax, (%r9)
1848 1848
1849 1849 / calculate the number of TS's that elapsed
1850 1850 movl %ecx, %eax
1851 1851 subl %esi, %eax
1852 1852 sbbl %edi, %edx
1853 1853
1854 1854 jmp freq_tsc_end
1855 1855
1856 1856 freq_tsc_too_fast:
1857 1857 / return 0 as a 64 bit quantity
1858 1858 xorl %eax, %eax
1859 1859 xorl %edx, %edx
1860 1860
1861 1861 freq_tsc_end:
1862 1862 shlq $32, %rdx
1863 1863 orq %rdx, %rax
1864 1864
1865 1865 popq %rbx
1866 1866 leaveq
1867 1867 ret
1868 1868 SET_SIZE(freq_tsc)
1869 1869
1870 1870 #elif defined(__i386)
1871 1871
1872 1872 ENTRY_NP(freq_tsc)
1873 1873 pushl %ebp
1874 1874 movl %esp, %ebp
1875 1875 pushl %edi
1876 1876 pushl %esi
1877 1877 pushl %ebx
1878 1878
1879 1879 / We have a TSC, but we have no way in general to know how reliable it is.
1880 1880 / Usually a marginal TSC behaves appropriately unless not enough time
1881 1881 / elapses between reads. A reliable TSC can be read as often and as rapidly
1882 1882 / as desired. The simplistic approach of reading the TSC counter and
1883 1883 / correlating to the PIT counter cannot be naively followed. Instead estimates
1884 1884 / have to be taken to successively refine a guess at the speed of the cpu
1885 1885 / and then the TSC and PIT counter are correlated. In practice very rarely
1886 1886 / is more than one quick loop required for an estimate. Measures have to be
1887 1887 / taken to prevent the PIT counter from wrapping beyond its resolution and for
1888 1888 / measuring the clock rate of very fast processors.
1889 1889 /
1890 1890 / The following constant can be tuned. It should be such that the loop does
1891 1891 / not take too many nor too few PIT counts to execute. If this value is too
1892 1892 / large, then on slow machines the loop will take a long time, or the PIT
1893 1893 / counter may even wrap. If this value is too small, then on fast machines
1894 1894 / the PIT counter may count so few ticks that the resolution of the PIT
1895 1895 / itself causes a bad guess. Because this code is used in machines with
1896 1896 / marginal TSC's and/or IO, if this value is too small on those, it may
1897 1897 / cause the calculated cpu frequency to vary slightly from boot to boot.
1898 1898 /
1899 1899 / In all cases even if this constant is set inappropriately, the algorithm
1900 1900 / will still work and the caller should be able to handle variances in the
1901 1901 / calculation of cpu frequency, but the calculation will be inefficient and
1902 1902 / take a disproportionate amount of time relative to a well selected value.
1903 1903 / As the slowest supported cpu becomes faster, this constant should be
1904 1904 / carefully increased.
1905 1905
1906 1906 movl $0x8000, %ecx
1907 1907
1908 1908 / to make sure the instruction cache has been warmed
1909 1909 clc
1910 1910
1911 1911 jmp freq_tsc_loop
1912 1912
1913 1913 / The following block of code up to and including the latching of the PIT
1914 1914 / counter after freq_tsc_perf_loop is very critical and very carefully
1915 1915 / written, it should only be modified with great care. freq_tsc_loop to
1916 1916 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
1917 1917 / freq_tsc_perf_loop up to the unlatching of the PIT counter.
1918 1918
1919 1919 .align 32
1920 1920 freq_tsc_loop:
1921 1921 / save the loop count in %ebx
1922 1922 movl %ecx, %ebx
1923 1923
1924 1924 / initialize the PIT counter and start a count down
1925 1925 movb $PIT_LOADMODE, %al
1926 1926 outb $PITCTL_PORT
1927 1927 movb $0xff, %al
1928 1928 outb $PITCTR0_PORT
1929 1929 outb $PITCTR0_PORT
1930 1930
1931 1931 / read the TSC and store the TS in %edi:%esi
1932 1932 rdtsc
1933 1933 movl %eax, %esi
1934 1934
1935 1935 freq_tsc_perf_loop:
1936 1936 movl %edx, %edi
1937 1937 movl %eax, %esi
1938 1938 movl %edx, %edi
1939 1939 loop freq_tsc_perf_loop
1940 1940
1941 1941 / read the TSC and store the LSW in %ecx
1942 1942 rdtsc
1943 1943 movl %eax, %ecx
1944 1944
1945 1945 / latch the PIT counter and status
1946 1946 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
1947 1947 outb $PITCTL_PORT
1948 1948
1949 1949 / remember if the icache has been warmed
1950 1950 setc %ah
1951 1951
1952 1952 / read the PIT status
1953 1953 inb $PITCTR0_PORT
1954 1954 shll $8, %eax
1955 1955
1956 1956 / read PIT count
1957 1957 inb $PITCTR0_PORT
1958 1958 shll $8, %eax
1959 1959 inb $PITCTR0_PORT
1960 1960 bswap %eax
1961 1961
1962 1962 / check to see if the PIT count was loaded into the CE
1963 1963 btw $_CONST(PITSTAT_NULLCNT+8), %ax
1964 1964 jc freq_tsc_increase_count
1965 1965
1966 1966 / check to see if PIT counter wrapped
1967 1967 btw $_CONST(PITSTAT_OUTPUT+8), %ax
1968 1968 jnc freq_tsc_pit_did_not_wrap
1969 1969
1970 1970 / halve count
1971 1971 shrl $1, %ebx
1972 1972 movl %ebx, %ecx
1973 1973
1974 1974 / the instruction cache has been warmed
1975 1975 stc
1976 1976
1977 1977 jmp freq_tsc_loop
1978 1978
1979 1979 freq_tsc_increase_count:
1980 1980 shll $1, %ebx
1981 1981 jc freq_tsc_too_fast
1982 1982
1983 1983 movl %ebx, %ecx
1984 1984
1985 1985 / the instruction cache has been warmed
1986 1986 stc
1987 1987
1988 1988 jmp freq_tsc_loop
1989 1989
1990 1990 freq_tsc_pit_did_not_wrap:
1991 1991 roll $16, %eax
1992 1992
1993 1993 cmpw $0x2000, %ax
1994 1994 notw %ax
1995 1995 jb freq_tsc_sufficient_duration
1996 1996
1997 1997 freq_tsc_calculate:
1998 1998 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
1999 1999 / then on the second CLK pulse the CE is decremented, therefore mode 0
2000 2000 / is really a (count + 1) counter, ugh
2001 2001 xorl %esi, %esi
2002 2002 movw %ax, %si
2003 2003 incl %esi
2004 2004
2005 2005 movl $0xf000, %eax
2006 2006 mull %ebx
2007 2007
2008 2008 / tuck away (target_pit_count * loop_count)
2009 2009 movl %edx, %ecx
2010 2010 movl %eax, %ebx
2011 2011
2012 2012 movl %esi, %eax
2013 2013 movl $0xffffffff, %edx
2014 2014 mull %edx
2015 2015
2016 2016 addl %esi, %eax
2017 2017 adcl $0, %edx
2018 2018
2019 2019 cmpl %ecx, %edx
2020 2020 ja freq_tsc_div_safe
2021 2021 jb freq_tsc_too_fast
2022 2022
2023 2023 cmpl %ebx, %eax
2024 2024 jbe freq_tsc_too_fast
2025 2025
2026 2026 freq_tsc_div_safe:
2027 2027 movl %ecx, %edx
2028 2028 movl %ebx, %eax
2029 2029
2030 2030 movl %esi, %ecx
2031 2031 divl %ecx
2032 2032
2033 2033 movl %eax, %ecx
2034 2034
2035 2035 / the instruction cache has been warmed
2036 2036 stc
2037 2037
2038 2038 jmp freq_tsc_loop
2039 2039
2040 2040 freq_tsc_sufficient_duration:
2041 2041 / test to see if the icache has been warmed
2042 2042 btl $16, %eax
2043 2043 jnc freq_tsc_calculate
2044 2044
2045 2045 / recall mode 0 is a (count + 1) counter
2046 2046 andl $0xffff, %eax
2047 2047 incl %eax
2048 2048
2049 2049 / save the number of PIT counts
2050 2050 movl 8(%ebp), %ebx
2051 2051 movl %eax, (%ebx)
2052 2052
2053 2053 / calculate the number of TS's that elapsed
2054 2054 movl %ecx, %eax
2055 2055 subl %esi, %eax
2056 2056 sbbl %edi, %edx
2057 2057
2058 2058 jmp freq_tsc_end
2059 2059
2060 2060 freq_tsc_too_fast:
2061 2061 / return 0 as a 64 bit quantity
2062 2062 xorl %eax, %eax
2063 2063 xorl %edx, %edx
2064 2064
2065 2065 freq_tsc_end:
2066 2066 popl %ebx
2067 2067 popl %esi
2068 2068 popl %edi
2069 2069 popl %ebp
2070 2070 ret
2071 2071 SET_SIZE(freq_tsc)
2072 2072
2073 2073 #endif /* __i386 */
2074 2074 #endif /* __lint */
2075 2075
2076 2076 #if !defined(__amd64)
2077 2077 #if defined(__lint)
2078 2078
2079 2079 /*
2080 2080 * We do not have a TSC so we use a block of instructions with well known
2081 2081 * timings.
2082 2082 */
2083 2083
2084 2084 /*ARGSUSED*/
2085 2085 uint64_t
2086 2086 freq_notsc(uint32_t *pit_counter)
2087 2087 {
2088 2088 return (0);
2089 2089 }
2090 2090
2091 2091 #else /* __lint */
2092 2092 ENTRY_NP(freq_notsc)
2093 2093 pushl %ebp
2094 2094 movl %esp, %ebp
2095 2095 pushl %edi
2096 2096 pushl %esi
2097 2097 pushl %ebx
2098 2098
2099 2099 / initial count for the idivl loop
2100 2100 movl $0x1000, %ecx
2101 2101
2102 2102 / load the divisor
2103 2103 movl $1, %ebx
2104 2104
2105 2105 jmp freq_notsc_loop
2106 2106
2107 2107 .align 16
2108 2108 freq_notsc_loop:
2109 2109 / set high 32 bits of dividend to zero
2110 2110 xorl %edx, %edx
2111 2111
2112 2112 / save the loop count in %edi
2113 2113 movl %ecx, %edi
2114 2114
2115 2115 / initialize the PIT counter and start a count down
2116 2116 movb $PIT_LOADMODE, %al
2117 2117 outb $PITCTL_PORT
2118 2118 movb $0xff, %al
2119 2119 outb $PITCTR0_PORT
2120 2120 outb $PITCTR0_PORT
2121 2121
2122 2122 / set low 32 bits of dividend to zero
2123 2123 xorl %eax, %eax
2124 2124
2125 2125 / It is vital that the arguments to idivl be set appropriately because on some
2126 2126 / cpu's this instruction takes more or less clock ticks depending on its
2127 2127 / arguments.
2128 2128 freq_notsc_perf_loop:
2129 2129 idivl %ebx
2130 2130 idivl %ebx
2131 2131 idivl %ebx
2132 2132 idivl %ebx
2133 2133 idivl %ebx
2134 2134 loop freq_notsc_perf_loop
2135 2135
2136 2136 / latch the PIT counter and status
2137 2137 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
2138 2138 outb $PITCTL_PORT
2139 2139
2140 2140 / read the PIT status
2141 2141 inb $PITCTR0_PORT
2142 2142 shll $8, %eax
2143 2143
2144 2144 / read PIT count
2145 2145 inb $PITCTR0_PORT
2146 2146 shll $8, %eax
2147 2147 inb $PITCTR0_PORT
2148 2148 bswap %eax
2149 2149
2150 2150 / check to see if the PIT count was loaded into the CE
2151 2151 btw $_CONST(PITSTAT_NULLCNT+8), %ax
2152 2152 jc freq_notsc_increase_count
2153 2153
2154 2154 / check to see if PIT counter wrapped
2155 2155 btw $_CONST(PITSTAT_OUTPUT+8), %ax
2156 2156 jnc freq_notsc_pit_did_not_wrap
2157 2157
2158 2158 / halve count
2159 2159 shrl $1, %edi
2160 2160 movl %edi, %ecx
2161 2161
2162 2162 jmp freq_notsc_loop
2163 2163
2164 2164 freq_notsc_increase_count:
2165 2165 shll $1, %edi
2166 2166 jc freq_notsc_too_fast
2167 2167
2168 2168 movl %edi, %ecx
2169 2169
2170 2170 jmp freq_notsc_loop
2171 2171
2172 2172 freq_notsc_pit_did_not_wrap:
2173 2173 shrl $16, %eax
2174 2174
2175 2175 cmpw $0x2000, %ax
2176 2176 notw %ax
2177 2177 jb freq_notsc_sufficient_duration
2178 2178
2179 2179 freq_notsc_calculate:
2180 2180 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
2181 2181 / then on the second CLK pulse the CE is decremented, therefore mode 0
2182 2182 / is really a (count + 1) counter, ugh
2183 2183 xorl %esi, %esi
2184 2184 movw %ax, %si
2185 2185 incl %esi
2186 2186
2187 2187 movl %edi, %eax
2188 2188 movl $0xf000, %ecx
2189 2189 mull %ecx
2190 2190
2191 2191 / tuck away (target_pit_count * loop_count)
2192 2192 movl %edx, %edi
2193 2193 movl %eax, %ecx
2194 2194
2195 2195 movl %esi, %eax
2196 2196 movl $0xffffffff, %edx
2197 2197 mull %edx
2198 2198
2199 2199 addl %esi, %eax
2200 2200 adcl $0, %edx
2201 2201
2202 2202 cmpl %edi, %edx
2203 2203 ja freq_notsc_div_safe
2204 2204 jb freq_notsc_too_fast
2205 2205
2206 2206 cmpl %ecx, %eax
2207 2207 jbe freq_notsc_too_fast
2208 2208
2209 2209 freq_notsc_div_safe:
2210 2210 movl %edi, %edx
2211 2211 movl %ecx, %eax
2212 2212
2213 2213 movl %esi, %ecx
2214 2214 divl %ecx
2215 2215
2216 2216 movl %eax, %ecx
2217 2217
2218 2218 jmp freq_notsc_loop
2219 2219
2220 2220 freq_notsc_sufficient_duration:
2221 2221 / recall mode 0 is a (count + 1) counter
2222 2222 incl %eax
2223 2223
2224 2224 / save the number of PIT counts
2225 2225 movl 8(%ebp), %ebx
2226 2226 movl %eax, (%ebx)
2227 2227
2228 2228 / calculate the number of cpu clock ticks that elapsed
2229 2229 cmpl $X86_VENDOR_Cyrix, x86_vendor
2230 2230 jz freq_notsc_notcyrix
2231 2231
2232 2232 / freq_notsc_perf_loop takes 86 clock cycles on Cyrix 6x86 cores
2233 2233 movl $86, %eax
2234 2234 jmp freq_notsc_calculate_tsc
2235 2235
2236 2236 freq_notsc_notcyrix:
2237 2237 / freq_notsc_perf_loop takes 237 clock cycles on Intel Pentiums
2238 2238 movl $237, %eax
2239 2239
2240 2240 freq_notsc_calculate_tsc:
2241 2241 mull %edi
2242 2242
2243 2243 jmp freq_notsc_end
2244 2244
2245 2245 freq_notsc_too_fast:
2246 2246 / return 0 as a 64 bit quantity
2247 2247 xorl %eax, %eax
2248 2248 xorl %edx, %edx
2249 2249
2250 2250 freq_notsc_end:
2251 2251 popl %ebx
2252 2252 popl %esi
2253 2253 popl %edi
2254 2254 popl %ebp
2255 2255
2256 2256 ret
2257 2257 SET_SIZE(freq_notsc)
2258 2258
2259 2259 #endif /* __lint */
2260 2260 #endif /* !__amd64 */
2261 2261
2262 2262 #if !defined(__lint)
2263 2263 .data
2264 2264 #if !defined(__amd64)
2265 2265 .align 4
2266 2266 cpu_vendor:
2267 2267 .long 0, 0, 0 /* Vendor ID string returned */
2268 2268
2269 2269 .globl CyrixInstead
2270 2270
2271 2271 .globl x86_featureset
2272 2272 .globl x86_type
2273 2273 .globl x86_vendor
2274 2274 #endif
2275 2275
2276 2276 #endif /* __lint */
|
↓ open down ↓ |
656 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX