Print this page
9685 KPTI %cr3 handling needs fixes
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/ml/locore.s
+++ new/usr/src/uts/i86pc/ml/locore.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25 /*
26 26 * Copyright (c) 2018 Joyent, Inc.
27 27 */
28 28
29 29 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
30 30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
31 31 /* All Rights Reserved */
32 32
33 33 /* Copyright (c) 1987, 1988 Microsoft Corporation */
34 34 /* All Rights Reserved */
35 35
36 36
37 37 #include <sys/asm_linkage.h>
38 38 #include <sys/asm_misc.h>
39 39 #include <sys/regset.h>
40 40 #include <sys/privregs.h>
41 41 #include <sys/psw.h>
42 42 #include <sys/reboot.h>
43 43 #include <sys/x86_archext.h>
44 44 #include <sys/machparam.h>
45 45
46 46 #if defined(__lint)
47 47
48 48 #include <sys/types.h>
49 49 #include <sys/thread.h>
50 50 #include <sys/systm.h>
51 51 #include <sys/lgrp.h>
52 52 #include <sys/regset.h>
53 53 #include <sys/link.h>
54 54 #include <sys/bootconf.h>
55 55 #include <sys/bootsvcs.h>
56 56
57 57 #else /* __lint */
58 58
59 59 #include <sys/segments.h>
60 60 #include <sys/pcb.h>
61 61 #include <sys/trap.h>
62 62 #include <sys/ftrace.h>
63 63 #include <sys/traptrace.h>
64 64 #include <sys/clock.h>
65 65 #include <sys/cmn_err.h>
66 66 #include <sys/pit.h>
67 67 #include <sys/panic.h>
68 68
69 69 #if defined(__xpv)
70 70 #include <sys/hypervisor.h>
71 71 #endif
72 72
73 73 #include "assym.h"
74 74
75 75 /*
76 76 * Our assumptions:
77 77 * - We are running in protected-paged mode.
78 78 * - Interrupts are disabled.
79 79 * - The GDT and IDT are the callers; we need our copies.
80 80 * - The kernel's text, initialized data and bss are mapped.
81 81 *
82 82 * Our actions:
83 83 * - Save arguments
84 84 * - Initialize our stack pointer to the thread 0 stack (t0stack)
85 85 * and leave room for a phony "struct regs".
86 86 * - Our GDT and IDT need to get munged.
87 87 * - Since we are using the boot's GDT descriptors, we need
88 88 * to copy them into our GDT before we switch to ours.
89 89 * - We start using our GDT by loading correct values in the
90 90 * selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
91 91 * gs=KGS_SEL).
92 92 * - The default LDT entry for syscall is set.
93 93 * - We load the default LDT into the hardware LDT register.
94 94 * - We load the default TSS into the hardware task register.
95 95 * - Check for cpu type, i.e. 486 vs. P5 vs. P6 etc.
96 96 * - mlsetup(%esp) gets called.
97 97 * - We change our appearance to look like the real thread 0.
98 98 * (NOTE: making ourselves to be a real thread may be a noop)
99 99 * - main() gets called. (NOTE: main() never returns).
100 100 *
101 101 * NOW, the real code!
102 102 */
103 103 /*
104 104 * The very first thing in the kernel's text segment must be a jump
105 105 * to the os/fakebop.c startup code.
106 106 */
107 107 .text
108 108 jmp _start
109 109
110 110 /*
111 111 * Globals:
112 112 */
113 113 .globl _locore_start
114 114 .globl mlsetup
115 115 .globl main
116 116 .globl panic
117 117 .globl t0stack
118 118 .globl t0
119 119 .globl sysp
120 120 .globl edata
121 121
122 122 /*
123 123 * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
124 124 */
125 125 .globl bootops
126 126 .globl bootopsp
127 127
128 128 /*
129 129 * NOTE: t0stack should be the first thing in the data section so that
130 130 * if it ever overflows, it will fault on the last kernel text page.
131 131 */
132 132 .data
133 133 .comm t0stack, DEFAULTSTKSZ, 32
134 134 .comm t0, 4094, 32
135 135
136 136 #endif /* __lint */
137 137
138 138
139 139 #if defined(__amd64)
140 140
141 141 #if defined(__lint)
142 142
143 143 /* ARGSUSED */
144 144 void
145 145 _locore_start(struct boot_syscalls *sysp, ulong_t rsi, struct bootops *bop)
146 146 {}
147 147
148 148 #else /* __lint */
149 149
150 150 /*
151 151 * kobj_init() vectors us back to here with (note) a slightly different
|
↓ open down ↓ |
151 lines elided |
↑ open up ↑ |
152 152 * set of arguments than _start is given (see lint prototypes above).
153 153 *
154 154 * XXX Make this less vile, please.
155 155 */
156 156 ENTRY_NP(_locore_start)
157 157
158 158 /*
159 159 * %rdi = boot services (should die someday)
160 160 * %rdx = bootops
161 161 * end
162 - */
162 + */
163 163
164 164 leaq edata(%rip), %rbp /* reference edata for ksyms */
165 165 movq $0, (%rbp) /* limit stack back trace */
166 166
167 167 /*
168 168 * Initialize our stack pointer to the thread 0 stack (t0stack)
169 169 * and leave room for a "struct regs" for lwp0. Note that the
170 170 * stack doesn't actually align to a 16-byte boundary until just
171 171 * before we call mlsetup because we want to use %rsp to point at
172 172 * our regs structure.
173 173 */
174 174 leaq t0stack(%rip), %rsp
175 175 addq $_CONST(DEFAULTSTKSZ - REGSIZE), %rsp
176 176 #if (REGSIZE & 15) == 0
177 177 subq $8, %rsp
178 178 #endif
179 179 /*
180 180 * Save call back for special x86 boot services vector
181 - */
181 + */
182 182 movq %rdi, sysp(%rip)
183 183
184 184 movq %rdx, bootops(%rip) /* save bootops */
185 185 movq $bootops, bootopsp(%rip)
186 186
187 187 /*
188 188 * Save arguments and flags, if only for debugging ..
189 189 */
190 190 movq %rdi, REGOFF_RDI(%rsp)
191 191 movq %rsi, REGOFF_RSI(%rsp)
192 192 movq %rdx, REGOFF_RDX(%rsp)
193 193 movq %rcx, REGOFF_RCX(%rsp)
194 194 movq %r8, REGOFF_R8(%rsp)
195 195 movq %r9, REGOFF_R9(%rsp)
196 196 pushf
197 197 popq %r11
198 198 movq %r11, REGOFF_RFL(%rsp)
199 199
200 200 #if !defined(__xpv)
|
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
201 201 /*
202 202 * Enable write protect and alignment check faults.
203 203 */
204 204 movq %cr0, %rax
205 205 orq $_CONST(CR0_WP|CR0_AM), %rax
206 206 andq $_BITNOT(CR0_WT|CR0_CE), %rax
207 207 movq %rax, %cr0
208 208 #endif /* __xpv */
209 209
210 210 /*
211 - * (We just assert this works by virtue of being here)
211 + * (We just assert this works by virtue of being here)
212 212 */
213 213 bts $X86FSET_CPUID, x86_featureset(%rip)
214 214
215 215 /*
216 216 * mlsetup() gets called with a struct regs as argument, while
217 217 * main takes no args and should never return.
218 218 */
219 219 xorl %ebp, %ebp
220 220 movq %rsp, %rdi
221 221 pushq %rbp
222 222 /* (stack pointer now aligned on 16-byte boundary right here) */
223 223 movq %rsp, %rbp
224 224 call mlsetup
225 225 call main
226 226 /* NOTREACHED */
227 227 leaq __return_from_main(%rip), %rdi
228 228 xorl %eax, %eax
229 229 call panic
230 230 SET_SIZE(_locore_start)
231 231
232 232 #endif /* __amd64 */
233 233 #endif /* __lint */
234 234
235 235 #if !defined(__lint)
236 236
237 237 __return_from_main:
238 238 .string "main() returned"
239 239 __unsupported_cpu:
240 240 .string "486 style cpu detected - no longer supported!"
241 241
242 242 #if defined(DEBUG)
243 243 _no_pending_updates:
244 244 .string "locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
245 245 #endif
246 246
247 247 #endif /* !__lint */
248 248
249 249 #if !defined(__amd64)
250 250
251 251 #if defined(__lint)
252 252
253 253 /* ARGSUSED */
254 254 void
255 255 _locore_start(struct boot_syscalls *sysp, struct bootops *bop)
256 256 {}
257 257
258 258 #else /* __lint */
259 259
260 260 /*
|
↓ open down ↓ |
39 lines elided |
↑ open up ↑ |
261 261 * kobj_init() vectors us back to here with (note) a slightly different
262 262 * set of arguments than _start is given (see lint prototypes above).
263 263 *
264 264 * XXX Make this less vile, please.
265 265 */
266 266 ENTRY_NP(_locore_start)
267 267
268 268 /*
269 269 * %ecx = boot services (should die someday)
270 270 * %ebx = bootops
271 - */
271 + */
272 272 mov $edata, %ebp / edata needs to be defined for ksyms
273 273 movl $0, (%ebp) / limit stack back trace
274 274
275 275 /*
276 276 * Initialize our stack pointer to the thread 0 stack (t0stack)
277 277 * and leave room for a phony "struct regs".
278 278 */
279 279 movl $t0stack + DEFAULTSTKSZ - REGSIZE, %esp
280 280
281 281 /*
282 282 * Save call back for special x86 boot services vector
283 283 */
284 284 mov %ecx, sysp / save call back for boot services
285 285
286 - mov %ebx, bootops / save bootops
286 + mov %ebx, bootops / save bootops
287 287 movl $bootops, bootopsp
288 288
289 289
290 290 /*
291 291 * Save all registers and flags
292 292 */
293 - pushal
293 + pushal
294 294 pushfl
295 295
296 296 #if !defined(__xpv)
297 297 /*
298 298 * Override bios settings and enable write protect and
299 299 * alignment check faults.
300 300 */
301 301 movl %cr0, %eax
302 302
303 303 /*
304 304 * enable WP for detecting faults, and enable alignment checking.
305 305 */
306 306 orl $_CONST(CR0_WP|CR0_AM), %eax
307 307 andl $_BITNOT(CR0_WT|CR0_CE), %eax
308 308 movl %eax, %cr0 / set the cr0 register correctly and
309 309 / override the BIOS setup
310 310
311 311 /*
312 312 * If bit 21 of eflags can be flipped, then cpuid is present
313 313 * and enabled.
314 314 */
315 315 pushfl
316 316 popl %ecx
317 317 movl %ecx, %eax
318 318 xorl $PS_ID, %eax / try complemented bit
319 319 pushl %eax
320 320 popfl
321 321 pushfl
322 322 popl %eax
323 323 cmpl %eax, %ecx
324 324 jne have_cpuid
325 325
326 326 /*
327 327 * cpuid may be disabled on Cyrix, try to detect Cyrix by the 5/2 test
328 328 * div does not modify the cc flags on Cyrix, even though this may
329 329 * also be true for other vendors, this is generally true only for
330 330 * newer models from those vendors that support and do not disable
331 331 * cpuid (usually because cpuid cannot be disabled)
332 332 */
333 333
334 334 /*
335 335 * clear cc flags
336 336 */
337 337 xorb %ah, %ah
338 338 sahf
339 339
340 340 /*
341 341 * perform 5/2 test
342 342 */
343 343 movw $5, %ax
344 344 movb $2, %bl
345 345 divb %bl
346 346
347 347 lahf
348 348 cmpb $2, %ah
349 349 jne cpu_486
350 350
351 351 /*
352 352 * div did not modify the cc flags, chances are the vendor is Cyrix
353 353 * assume the vendor is Cyrix and use the CCR's to enable cpuid
354 354 */
355 355 .set CYRIX_CRI, 0x22 / CR Index Register
356 356 .set CYRIX_CRD, 0x23 / CR Data Register
357 357
358 358 .set CYRIX_CCR3, 0xc3 / Config Control Reg 3
359 359 .set CYRIX_CCR4, 0xe8 / Config Control Reg 4
360 360 .set CYRIX_DIR0, 0xfe / Device Identification Reg 0
361 361 .set CYRIX_DIR1, 0xff / Device Identification Reg 1
362 362
363 363 /*
364 364 * even if the cpu vendor is Cyrix and the motherboard/chipset
365 365 * vendor decided to ignore lines A1-A4 for I/O addresses, I/O port
366 366 * 0x21 corresponds with 0x23 and since 0x22 is still untouched,
367 367 * the reads and writes of 0x21 are guaranteed to be off-chip of
368 368 * the cpu
369 369 */
370 370
371 371 /*
372 372 * enable read of ISR at I/O port 0x20
373 373 */
374 374 movb $0xb, %al
375 375 outb $MCMD_PORT
376 376
377 377 /*
378 378 * read IMR and store in %bl
379 379 */
380 380 inb $MIMR_PORT
381 381 movb %al, %bl
382 382
383 383 /*
384 384 * mask out all interrupts so that ISR will not change
385 385 */
386 386 movb $0xff, %al
387 387 outb $MIMR_PORT
388 388
389 389 /*
390 390 * reads of I/O port 0x22 on Cyrix are always directed off-chip
391 391 * make use of I/O pull-up to test for an unknown device on 0x22
392 392 */
393 393 inb $CYRIX_CRI
394 394 cmpb $0xff, %al
395 395 je port_22_free
396 396
397 397 /*
398 398 * motherboard/chipset vendor may be ignoring line A1 of I/O address
399 399 */
400 400 movb %al, %cl
401 401
402 402 /*
403 403 * if the ISR and the value read from 0x22 do not match then we have
404 404 * detected some unknown device, probably a chipset, at 0x22
405 405 */
406 406 inb $MCMD_PORT
407 407 cmpb %al, %cl
408 408 jne restore_IMR
409 409
410 410 port_22_free:
411 411 /*
412 412 * now test to see if some unknown device is using I/O port 0x23
413 413 *
414 414 * read the external I/O port at 0x23
415 415 */
416 416 inb $CYRIX_CRD
417 417
418 418 /*
419 419 * Test for pull-up at 0x23 or if I/O address line A1 is being ignored.
420 420 * IMR is 0xff so both tests are performed simultaneously.
421 421 */
422 422 cmpb $0xff, %al
423 423 jne restore_IMR
424 424
425 425 /*
426 426 * We are a Cyrix part. In case we are some model of Cx486 or a Cx586,
427 427 * record the type and fix it later if not.
428 428 */
429 429 movl $X86_VENDOR_Cyrix, x86_vendor
430 430 movl $X86_TYPE_CYRIX_486, x86_type
431 431
432 432 /*
433 433 * Try to read CCR3. All Cyrix cpu's which support cpuid have CCR3.
434 434 *
435 435 * load CCR3 index into CCR index register
|
↓ open down ↓ |
132 lines elided |
↑ open up ↑ |
436 436 */
437 437
438 438 movb $CYRIX_CCR3, %al
439 439 outb $CYRIX_CRI
440 440
441 441 /*
442 442 * If we are not a Cyrix cpu, then we have performed an external I/O
443 443 * cycle. If the CCR index was not valid for this Cyrix model, we may
444 444 * have performed an external I/O cycle as well. In these cases and
445 445 * if the motherboard/chipset vendor ignores I/O address line A1,
446 - * then the PIC will have IRQ3 set at the lowest priority as a side
446 + * then the PIC will have IRQ3 set at the lowest priority as a side
447 447 * effect of the above outb. We are reasonalbly confident that there
448 448 * is not an unknown device on I/O port 0x22, so there should have been
449 449 * no unpredictable side-effect of the above outb.
450 450 */
451 451
452 452 /*
453 453 * read CCR3
454 454 */
455 455 inb $CYRIX_CRD
456 456
457 457 /*
458 458 * If we are not a Cyrix cpu the inb above produced an external I/O
459 459 * cycle. If we are a Cyrix model that does not support CCR3 wex
460 460 * produced an external I/O cycle. In all known Cyrix models 6x86 and
461 461 * above, bit 3 of CCR3 is reserved and cannot be set to 1. In all
462 462 * Cyrix models prior to the 6x86 that supported CCR3, bits 4-7 are
463 463 * reserved as well. It is highly unlikely that CCR3 contains the value
464 464 * 0xff. We test to see if I/O port 0x23 is pull-up or the IMR and
465 465 * deduce we are not a Cyrix with support for cpuid if so.
466 466 */
467 467 cmpb $0xff, %al
468 468 je restore_PIC
469 469
470 470 /*
471 471 * There exist 486 ISA Cyrix chips that support CCR3 but do not support
472 472 * DIR0 and DIR1. If we try to read DIR0, we may generate external I/O
473 473 * cycles, the exact behavior is model specific and undocumented.
474 474 * Unfortunately these external I/O cycles may confuse some PIC's beyond
475 475 * recovery. Fortunatetly we can use the following undocumented trick:
476 476 * if bit 4 of CCR3 can be toggled, then DIR0 and DIR1 are supported.
477 477 * Pleasantly MAPEN contains bit 4 of CCR3, so this trick is guaranteed
478 478 * to work on all Cyrix cpu's which support cpuid.
479 479 */
480 480 movb %al, %dl
481 481 xorb $0x10, %dl
482 482 movb %al, %cl
483 483
484 484 /*
485 485 * write back CRR3 with toggled bit 4 to CCR3
486 486 */
487 487 movb $CYRIX_CCR3, %al
488 488 outb $CYRIX_CRI
489 489
490 490 movb %dl, %al
491 491 outb $CYRIX_CRD
492 492
493 493 /*
494 494 * read CCR3
495 495 */
496 496 movb $CYRIX_CCR3, %al
497 497 outb $CYRIX_CRI
498 498 inb $CYRIX_CRD
499 499 movb %al, %dl
500 500
501 501 /*
502 502 * restore CCR3
503 503 */
504 504 movb $CYRIX_CCR3, %al
505 505 outb $CYRIX_CRI
506 506
507 507 movb %cl, %al
508 508 outb $CYRIX_CRD
509 509
510 510 /*
511 511 * if bit 4 was not toggled DIR0 and DIR1 are not supported in which
512 512 * case we do not have cpuid anyway
513 513 */
514 514 andb $0x10, %al
515 515 andb $0x10, %dl
516 516 cmpb %al, %dl
517 517 je restore_PIC
518 518
519 519 /*
520 520 * read DIR0
521 521 */
522 522 movb $CYRIX_DIR0, %al
523 523 outb $CYRIX_CRI
524 524 inb $CYRIX_CRD
525 525
526 526 /*
527 527 * test for pull-up
528 528 */
529 529 cmpb $0xff, %al
530 530 je restore_PIC
531 531
532 532 /*
533 533 * Values of 0x20-0x27 in DIR0 are currently reserved by Cyrix for
534 534 * future use. If Cyrix ever produces a cpu that supports cpuid with
535 535 * these ids, the following test will have to change. For now we remain
536 536 * pessimistic since the formats of the CRR's may be different then.
537 537 *
538 538 * test for at least a 6x86, to see if we support both MAPEN and CPUID
539 539 */
540 540 cmpb $0x30, %al
541 541 jb restore_IMR
542 542
543 543 /*
544 544 * enable MAPEN
545 545 */
546 546 movb $CYRIX_CCR3, %al
547 547 outb $CYRIX_CRI
548 548
549 549 andb $0xf, %cl
550 550 movb %cl, %al
551 551 orb $0x10, %al
552 552 outb $CYRIX_CRD
553 553
554 554 /*
555 555 * select CCR4
556 556 */
557 557 movb $CYRIX_CCR4, %al
558 558 outb $CYRIX_CRI
559 559
560 560 /*
561 561 * read CCR4
562 562 */
563 563 inb $CYRIX_CRD
564 564
565 565 /*
566 566 * enable cpuid
567 567 */
568 568 orb $0x80, %al
569 569 movb %al, %dl
570 570
571 571 /*
572 572 * select CCR4
573 573 */
574 574 movb $CYRIX_CCR4, %al
575 575 outb $CYRIX_CRI
576 576
577 577 /*
578 578 * write CCR4
579 579 */
580 580 movb %dl, %al
581 581 outb $CYRIX_CRD
582 582
583 583 /*
584 584 * select CCR3
585 585 */
586 586 movb $CYRIX_CCR3, %al
587 587 outb $CYRIX_CRI
588 588
589 589 /*
590 590 * disable MAPEN and write CCR3
591 591 */
592 592 movb %cl, %al
593 593 outb $CYRIX_CRD
594 594
595 595 /*
596 596 * restore IMR
597 597 */
598 598 movb %bl, %al
599 599 outb $MIMR_PORT
600 600
601 601 /*
602 602 * test to see if cpuid available
603 603 */
604 604 pushfl
605 605 popl %ecx
606 606 movl %ecx, %eax
607 607 xorl $PS_ID, %eax / try complemented bit
608 608 pushl %eax
609 609 popfl
610 610 pushfl
611 611 popl %eax
612 612 cmpl %eax, %ecx
613 613 jne have_cpuid
614 614 jmp cpu_486
615 615
616 616 restore_PIC:
617 617 /*
618 618 * In case the motherboard/chipset vendor is ignoring line A1 of the
619 619 * I/O address, we set the PIC priorities to sane values.
620 620 */
621 621 movb $0xc7, %al / irq 7 lowest priority
622 622 outb $MCMD_PORT
623 623
624 624 restore_IMR:
625 625 movb %bl, %al
626 626 outb $MIMR_PORT
627 627 jmp cpu_486
628 628
629 629 have_cpuid:
630 630 /*
631 631 * cpuid instruction present
632 632 */
633 633 bts $X86FSET_CPUID, x86_featureset / Just to set; Ignore the CF
634 634 movl $0, %eax
635 635 cpuid
636 636
637 637 movl %ebx, cpu_vendor
638 638 movl %edx, cpu_vendor+4
639 639 movl %ecx, cpu_vendor+8
640 640
641 641 /*
642 642 * early cyrix cpus are somewhat strange and need to be
643 643 * probed in curious ways to determine their identity
644 644 */
645 645
646 646 leal cpu_vendor, %esi
647 647 leal CyrixInstead, %edi
648 648 movl $12, %ecx
649 649 repz
650 650 cmpsb
651 651 je vendor_is_cyrix
652 652
653 653 / let mlsetup()/cpuid_pass1() handle everything else in C
654 654
655 655 jmp cpu_done
656 656
657 657 is486:
658 658 /*
659 659 * test to see if a useful cpuid
660 660 */
661 661 testl %eax, %eax
662 662 jz isa486
663 663
664 664 movl $1, %eax
665 665 cpuid
666 666
667 667 movl %eax, %ebx
668 668 andl $0xF00, %ebx
669 669 cmpl $0x400, %ebx
670 670 je isa486
671 671
672 672 rep; ret /* use 2 byte return instruction */
673 673 /* AMD Software Optimization Guide - Section 6.2 */
674 674 isa486:
675 675 /*
676 676 * lose the return address
677 677 */
678 678 popl %eax
679 679 jmp cpu_486
680 680
681 681 vendor_is_cyrix:
682 682 call is486
683 683
684 684 /*
685 685 * Processor signature and feature flags for Cyrix are insane.
686 686 * BIOS can play with semi-documented registers, so cpuid must be used
687 687 * cautiously. Since we are Cyrix that has cpuid, we have DIR0 and DIR1
688 688 * Keep the family in %ebx and feature flags in %edx until not needed
689 689 */
690 690
691 691 /*
692 692 * read DIR0
693 693 */
694 694 movb $CYRIX_DIR0, %al
695 695 outb $CYRIX_CRI
696 696 inb $CYRIX_CRD
697 697
698 698 /*
699 699 * First we handle the cases where we are a 6x86 or 6x86L.
700 700 * The 6x86 is basically a 486, the only reliable bit in the
701 701 * feature flags is for FPU. The 6x86L is better, unfortunately
702 702 * there is no really good way to distinguish between these two
703 703 * cpu's. We are pessimistic and when in doubt assume 6x86.
704 704 */
705 705
706 706 cmpb $0x40, %al
707 707 jae maybeGX
708 708
709 709 /*
710 710 * We are an M1, either a 6x86 or 6x86L.
711 711 */
712 712 cmpb $0x30, %al
713 713 je maybe6x86L
714 714 cmpb $0x31, %al
715 715 je maybe6x86L
716 716 cmpb $0x34, %al
717 717 je maybe6x86L
718 718 cmpb $0x35, %al
719 719 je maybe6x86L
720 720
721 721 /*
722 722 * although it is possible that we are a 6x86L, the cpu and
723 723 * documentation are so buggy, we just do not care.
724 724 */
725 725 jmp likely6x86
726 726
727 727 maybe6x86L:
728 728 /*
729 729 * read DIR1
730 730 */
731 731 movb $CYRIX_DIR1, %al
732 732 outb $CYRIX_CRI
733 733 inb $CYRIX_CRD
734 734 cmpb $0x22, %al
735 735 jb likely6x86
736 736
737 737 /*
738 738 * We are a 6x86L, or at least a 6x86 with honest cpuid feature flags
739 739 */
740 740 movl $X86_TYPE_CYRIX_6x86L, x86_type
741 741 jmp coma_bug
742 742
743 743 likely6x86:
744 744 /*
745 745 * We are likely a 6x86, or a 6x86L without a way of knowing
746 746 *
747 747 * The 6x86 has NO Pentium or Pentium Pro compatible features even
748 748 * though it claims to be a Pentium Pro compatible!
749 749 *
750 750 * The 6x86 core used in the 6x86 may have most of the Pentium system
751 751 * registers and largely conform to the Pentium System Programming
752 752 * Reference. Documentation on these parts is long gone. Treat it as
753 753 * a crippled Pentium and hope for the best.
754 754 */
755 755
756 756 movl $X86_TYPE_CYRIX_6x86, x86_type
757 757 jmp coma_bug
758 758
759 759 maybeGX:
760 760 /*
761 761 * Now we check whether we are a MediaGX or GXm. We have particular
762 762 * reason for concern here. Even though most of the GXm's
763 763 * report having TSC in the cpuid feature flags, the TSC may be
764 764 * horribly broken. What is worse, is that MediaGX's are basically
765 765 * 486's while the good GXm's are more like Pentium Pro's!
766 766 */
767 767
768 768 cmpb $0x50, %al
769 769 jae maybeM2
770 770
771 771 /*
772 772 * We are either a MediaGX (sometimes called a Gx86) or GXm
773 773 */
774 774
775 775 cmpb $41, %al
776 776 je maybeMediaGX
777 777
778 778 cmpb $44, %al
779 779 jb maybeGXm
780 780
781 781 cmpb $47, %al
782 782 jbe maybeMediaGX
783 783
784 784 /*
785 785 * We do not honestly know what we are, so assume a MediaGX
786 786 */
787 787 jmp media_gx
788 788
789 789 maybeGXm:
790 790 /*
791 791 * It is still possible we are either a MediaGX or GXm, trust cpuid
792 792 * family should be 5 on a GXm
793 793 */
794 794 cmpl $0x500, %ebx
795 795 je GXm
796 796
797 797 /*
798 798 * BIOS/Cyrix might set family to 6 on a GXm
799 799 */
800 800 cmpl $0x600, %ebx
801 801 jne media_gx
802 802
803 803 GXm:
804 804 movl $X86_TYPE_CYRIX_GXm, x86_type
805 805 jmp cpu_done
806 806
807 807 maybeMediaGX:
808 808 /*
809 809 * read DIR1
810 810 */
811 811 movb $CYRIX_DIR1, %al
812 812 outb $CYRIX_CRI
813 813 inb $CYRIX_CRD
814 814
815 815 cmpb $0x30, %al
816 816 jae maybeGXm
817 817
818 818 /*
819 819 * we are a MediaGX for which we do not trust cpuid
820 820 */
821 821 media_gx:
822 822 movl $X86_TYPE_CYRIX_MediaGX, x86_type
823 823 jmp cpu_486
824 824
825 825 maybeM2:
826 826 /*
827 827 * Now we check whether we are a 6x86MX or MII. These cpu's are
828 828 * virtually identical, but we care because for the 6x86MX, we
829 829 * must work around the coma bug. Also for 6x86MX prior to revision
830 830 * 1.4, the TSC may have serious bugs.
831 831 */
832 832
833 833 cmpb $0x60, %al
834 834 jae maybeM3
835 835
836 836 /*
837 837 * family should be 6, but BIOS/Cyrix might set it to 5
838 838 */
839 839 cmpl $0x600, %ebx
840 840 ja cpu_486
841 841
842 842 /*
843 843 * read DIR1
844 844 */
845 845 movb $CYRIX_DIR1, %al
846 846 outb $CYRIX_CRI
847 847 inb $CYRIX_CRD
848 848
849 849 cmpb $0x8, %al
850 850 jb cyrix6x86MX
851 851 cmpb $0x80, %al
852 852 jb MII
853 853
854 854 cyrix6x86MX:
855 855 /*
856 856 * It is altogether unclear how the revision stamped on the cpu
857 857 * maps to the values in DIR0 and DIR1. Just assume TSC is broken.
858 858 */
859 859 movl $X86_TYPE_CYRIX_6x86MX, x86_type
860 860 jmp coma_bug
861 861
862 862 MII:
863 863 movl $X86_TYPE_CYRIX_MII, x86_type
864 864 likeMII:
865 865 jmp cpu_done
866 866
867 867 maybeM3:
868 868 /*
869 869 * We are some chip that we cannot identify yet, an MIII perhaps.
870 870 * We will be optimistic and hope that the chip is much like an MII,
871 871 * and that cpuid is sane. Cyrix seemed to have gotten it right in
872 872 * time for the MII, we can only hope it stayed that way.
873 873 * Maybe the BIOS or Cyrix is trying to hint at something
874 874 */
875 875 cmpl $0x500, %ebx
876 876 je GXm
877 877
878 878 cmpb $0x80, %al
879 879 jae likelyM3
880 880
881 881 /*
882 882 * Just test for the features Cyrix is known for
883 883 */
884 884
|
↓ open down ↓ |
428 lines elided |
↑ open up ↑ |
885 885 jmp MII
886 886
887 887 likelyM3:
888 888 /*
889 889 * DIR0 with values from 0x80 to 0x8f indicates a VIA Cyrix III, aka
890 890 * the Cyrix MIII. There may be parts later that use the same ranges
891 891 * for DIR0 with special values in DIR1, maybe the VIA CIII, but for
892 892 * now we will call anything with a DIR0 of 0x80 or higher an MIII.
893 893 * The MIII is supposed to support large pages, but we will believe
894 894 * it when we see it. For now we just enable and test for MII features.
895 - */
895 + */
896 896 movl $X86_TYPE_VIA_CYRIX_III, x86_type
897 897 jmp likeMII
898 898
899 899 coma_bug:
900 900
901 901 /*
902 902 * With NO_LOCK set to 0 in CCR1, the usual state that BIOS enforces, some
903 903 * bus cycles are issued with LOCK# asserted. With NO_LOCK set to 1, all bus
904 904 * cycles except page table accesses and interrupt ACK cycles do not assert
905 905 * LOCK#. xchgl is an instruction that asserts LOCK# if NO_LOCK is set to 0.
906 906 * Due to a bug in the cpu core involving over-optimization of branch
907 907 * prediction, register renaming, and execution of instructions down both the
908 908 * X and Y pipes for the xchgl instruction, short loops can be written that
909 909 * never de-assert LOCK# from one invocation of the loop to the next, ad
910 910 * infinitum. The undesirable effect of this situation is that interrupts are
911 911 * not serviced. The ideal workaround to this bug would be to set NO_LOCK to
912 912 * 1. Unfortunately bus cycles that would otherwise have asserted LOCK# no
913 913 * longer do, unless they are page table accesses or interrupt ACK cycles.
914 914 * With LOCK# not asserted, these bus cycles are now cached. This can cause
915 915 * undesirable behaviour if the ARR's are not configured correctly. Solaris
916 916 * does not configure the ARR's, nor does it provide any useful mechanism for
917 917 * doing so, thus the ideal workaround is not viable. Fortunately, the only
918 918 * known exploits for this bug involve the xchgl instruction specifically.
919 919 * There is a group of undocumented registers on Cyrix 6x86, 6x86L, and
920 920 * 6x86MX cpu's which can be used to specify one instruction as a serializing
921 921 * instruction. With the xchgl instruction serialized, LOCK# is still
922 922 * asserted, but it is the sole instruction for which LOCK# is asserted.
|
↓ open down ↓ |
17 lines elided |
↑ open up ↑ |
923 923 * There is now some added penalty for the xchgl instruction, but the usual
924 924 * bus locking is preserved. This ingenious workaround was discovered by
925 925 * disassembling a binary provided by Cyrix as a workaround for this bug on
926 926 * Windows, but its not documented anywhere by Cyrix, nor is the bug actually
927 927 * mentioned in any public errata! The only concern for this workaround is
928 928 * that there may be similar undiscovered bugs with other instructions that
929 929 * assert LOCK# that may be leveraged to similar ends. The fact that Cyrix
930 930 * fixed this bug sometime late in 1997 and no other exploits other than
931 931 * xchgl have been discovered is good indication that this workaround is
932 932 * reasonable.
933 - */
933 + */
934 934
935 935 .set CYRIX_DBR0, 0x30 / Debug Register 0
936 936 .set CYRIX_DBR1, 0x31 / Debug Register 1
937 937 .set CYRIX_DBR2, 0x32 / Debug Register 2
938 938 .set CYRIX_DBR3, 0x33 / Debug Register 3
939 939 .set CYRIX_DOR, 0x3c / Debug Opcode Register
940 940
941 941 /*
942 - * What is known about DBR1, DBR2, DBR3, and DOR is that for normal
942 + * What is known about DBR1, DBR2, DBR3, and DOR is that for normal
943 943 * cpu execution DBR1, DBR2, and DBR3 are set to 0. To obtain opcode
944 944 * serialization, DBR1, DBR2, and DBR3 are loaded with 0xb8, 0x7f,
945 945 * and 0xff. Then, DOR is loaded with the one byte opcode.
946 946 */
947 947
948 948 /*
949 949 * select CCR3
950 950 */
951 951 movb $CYRIX_CCR3, %al
952 952 outb $CYRIX_CRI
953 953
954 954 /*
955 955 * read CCR3 and mask out MAPEN
956 956 */
957 957 inb $CYRIX_CRD
958 958 andb $0xf, %al
959 959
960 960 /*
961 961 * save masked CCR3 in %ah
962 962 */
963 963 movb %al, %ah
964 964
965 965 /*
966 966 * select CCR3
967 967 */
968 968 movb $CYRIX_CCR3, %al
969 969 outb $CYRIX_CRI
970 970
971 971 /*
972 972 * enable MAPEN
973 973 */
974 974 movb %ah, %al
975 975 orb $0x10, %al
976 976 outb $CYRIX_CRD
977 977
978 978 /*
979 979 * read DBR0
980 980 */
981 981 movb $CYRIX_DBR0, %al
982 982 outb $CYRIX_CRI
983 983 inb $CYRIX_CRD
984 984
985 985 /*
986 986 * disable MATCH and save in %bh
987 987 */
988 988 orb $0x80, %al
989 989 movb %al, %bh
990 990
991 991 /*
|
↓ open down ↓ |
39 lines elided |
↑ open up ↑ |
992 992 * write DBR0
993 993 */
994 994 movb $CYRIX_DBR0, %al
995 995 outb $CYRIX_CRI
996 996 movb %bh, %al
997 997 outb $CYRIX_CRD
998 998
999 999 /*
1000 1000 * write DBR1
1001 1001 */
1002 - movb $CYRIX_DBR1, %al
1002 + movb $CYRIX_DBR1, %al
1003 1003 outb $CYRIX_CRI
1004 1004 movb $0xf8, %al
1005 1005 outb $CYRIX_CRD
1006 1006
1007 1007 /*
1008 1008 * write DBR2
1009 1009 */
1010 1010 movb $CYRIX_DBR2, %al
1011 1011 outb $CYRIX_CRI
1012 1012 movb $0x7f, %al
1013 1013 outb $CYRIX_CRD
1014 1014
1015 1015 /*
1016 1016 * write DBR3
1017 1017 */
1018 1018 movb $CYRIX_DBR3, %al
1019 1019 outb $CYRIX_CRI
1020 1020 xorb %al, %al
1021 1021 outb $CYRIX_CRD
1022 1022
1023 1023 /*
1024 1024 * write DOR
1025 1025 */
1026 1026 movb $CYRIX_DOR, %al
1027 1027 outb $CYRIX_CRI
1028 1028 movb $0x87, %al
1029 1029 outb $CYRIX_CRD
1030 1030
1031 1031 /*
1032 1032 * enable MATCH
1033 1033 */
1034 1034 movb $CYRIX_DBR0, %al
1035 1035 outb $CYRIX_CRI
1036 1036 movb %bh, %al
1037 1037 andb $0x7f, %al
1038 1038 outb $CYRIX_CRD
1039 1039
1040 1040 /*
1041 1041 * disable MAPEN
1042 1042 */
1043 1043 movb $0xc3, %al
1044 1044 outb $CYRIX_CRI
1045 1045 movb %ah, %al
1046 1046 outb $CYRIX_CRD
1047 1047
1048 1048 jmp cpu_done
1049 1049
1050 1050 cpu_done:
1051 1051
1052 1052 popfl /* Restore original FLAGS */
1053 1053 popal /* Restore all registers */
1054 1054
1055 1055 #endif /* !__xpv */
1056 1056
1057 1057 /*
1058 1058 * mlsetup(%esp) gets called.
1059 1059 */
1060 1060 pushl %esp
1061 1061 call mlsetup
1062 1062 addl $4, %esp
1063 1063
1064 1064 /*
1065 1065 * We change our appearance to look like the real thread 0.
1066 1066 * (NOTE: making ourselves to be a real thread may be a noop)
1067 1067 * main() gets called. (NOTE: main() never returns).
1068 1068 */
1069 1069 call main
1070 1070 /* NOTREACHED */
1071 1071 pushl $__return_from_main
1072 1072 call panic
1073 1073
1074 1074 /* NOTREACHED */
1075 1075 cpu_486:
1076 1076 pushl $__unsupported_cpu
1077 1077 call panic
1078 1078 SET_SIZE(_locore_start)
1079 1079
1080 1080 #endif /* __lint */
1081 1081 #endif /* !__amd64 */
1082 1082
1083 1083
1084 1084 /*
1085 1085 * For stack layout, see privregs.h
1086 1086 * When cmntrap gets called, the error code and trap number have been pushed.
1087 1087 * When cmntrap_pushed gets called, the entire struct regs has been pushed.
1088 1088 */
1089 1089
1090 1090 #if defined(__lint)
1091 1091
1092 1092 /* ARGSUSED */
1093 1093 void
1094 1094 cmntrap()
1095 1095 {}
1096 1096
1097 1097 #else /* __lint */
1098 1098
1099 1099 .globl trap /* C handler called below */
1100 1100
1101 1101 #if defined(__amd64)
1102 1102
1103 1103 ENTRY_NP2(cmntrap, _cmntrap)
1104 1104
1105 1105 INTR_PUSH
1106 1106
1107 1107 ALTENTRY(cmntrap_pushed)
1108 1108
1109 1109 movq %rsp, %rbp
1110 1110
1111 1111 /*
1112 1112 * - if this is a #pf i.e. T_PGFLT, %r15 is live
1113 1113 * and contains the faulting address i.e. a copy of %cr2
1114 1114 *
1115 1115 * - if this is a #db i.e. T_SGLSTP, %r15 is live
1116 1116 * and contains the value of %db6
1117 1117 */
1118 1118
1119 1119 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1120 1120 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1121 1121 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1122 1122
1123 1123 /*
1124 1124 * We must first check if DTrace has set its NOFAULT bit. This
1125 1125 * regrettably must happen before the trap stack is recorded, because
1126 1126 * this requires a call to getpcstack() and may induce recursion if an
1127 1127 * fbt::getpcstack: enabling is inducing the bad load.
1128 1128 */
1129 1129 movl %gs:CPU_ID, %eax
1130 1130 shlq $CPU_CORE_SHIFT, %rax
1131 1131 leaq cpu_core(%rip), %r8
1132 1132 addq %r8, %rax
1133 1133 movw CPUC_DTRACE_FLAGS(%rax), %cx
1134 1134 testw $CPU_DTRACE_NOFAULT, %cx
1135 1135 jnz .dtrace_induced
1136 1136
1137 1137 TRACE_STACK(%rdi)
1138 1138
1139 1139 movq %rbp, %rdi
1140 1140 movq %r15, %rsi
1141 1141 movl %gs:CPU_ID, %edx
1142 1142
1143 1143 /*
1144 1144 * We know that this isn't a DTrace non-faulting load; we can now safely
1145 1145 * reenable interrupts. (In the case of pagefaults, we enter through an
1146 1146 * interrupt gate.)
1147 1147 */
1148 1148 ENABLE_INTR_FLAGS
1149 1149
1150 1150 call trap /* trap(rp, addr, cpuid) handles all traps */
1151 1151 jmp _sys_rtt
1152 1152
1153 1153 .dtrace_induced:
1154 1154 cmpw $KCS_SEL, REGOFF_CS(%rbp) /* test CS for user-mode trap */
1155 1155 jne 3f /* if from user, panic */
1156 1156
1157 1157 cmpl $T_PGFLT, REGOFF_TRAPNO(%rbp)
1158 1158 je 1f
1159 1159
1160 1160 cmpl $T_GPFLT, REGOFF_TRAPNO(%rbp)
1161 1161 je 0f
1162 1162
1163 1163 cmpl $T_ILLINST, REGOFF_TRAPNO(%rbp)
1164 1164 je 0f
1165 1165
1166 1166 cmpl $T_ZERODIV, REGOFF_TRAPNO(%rbp)
1167 1167 jne 4f /* if not PF/GP/UD/DE, panic */
1168 1168
1169 1169 orw $CPU_DTRACE_DIVZERO, %cx
1170 1170 movw %cx, CPUC_DTRACE_FLAGS(%rax)
1171 1171 jmp 2f
1172 1172
1173 1173 /*
1174 1174 * If we've taken a GPF, we don't (unfortunately) have the address that
1175 1175 * induced the fault. So instead of setting the fault to BADADDR,
1176 1176 * we'll set the fault to ILLOP.
1177 1177 */
1178 1178 0:
1179 1179 orw $CPU_DTRACE_ILLOP, %cx
1180 1180 movw %cx, CPUC_DTRACE_FLAGS(%rax)
1181 1181 jmp 2f
1182 1182 1:
1183 1183 orw $CPU_DTRACE_BADADDR, %cx
1184 1184 movw %cx, CPUC_DTRACE_FLAGS(%rax) /* set fault to bad addr */
1185 1185 movq %r15, CPUC_DTRACE_ILLVAL(%rax)
1186 1186 /* fault addr is illegal value */
1187 1187 2:
1188 1188 movq REGOFF_RIP(%rbp), %rdi
1189 1189 movq %rdi, %r12
1190 1190 call dtrace_instr_size
1191 1191 addq %rax, %r12
1192 1192 movq %r12, REGOFF_RIP(%rbp)
1193 1193 INTR_POP
|
↓ open down ↓ |
181 lines elided |
↑ open up ↑ |
1194 1194 jmp tr_iret_auto
1195 1195 /*NOTREACHED*/
1196 1196 3:
1197 1197 leaq dtrace_badflags(%rip), %rdi
1198 1198 xorl %eax, %eax
1199 1199 call panic
1200 1200 4:
1201 1201 leaq dtrace_badtrap(%rip), %rdi
1202 1202 xorl %eax, %eax
1203 1203 call panic
1204 + SET_SIZE(cmntrap_pushed)
1204 1205 SET_SIZE(cmntrap)
1205 1206 SET_SIZE(_cmntrap)
1206 1207
1207 1208 #elif defined(__i386)
1208 1209
1209 1210
1210 1211 ENTRY_NP2(cmntrap, _cmntrap)
1211 1212
1212 1213 INTR_PUSH
1213 1214
1214 1215 ALTENTRY(cmntrap_pushed)
1215 1216
1216 1217 movl %esp, %ebp
1217 1218
1218 1219 /*
1219 1220 * - if this is a #pf i.e. T_PGFLT, %esi is live
1220 1221 * and contains the faulting address i.e. a copy of %cr2
1221 1222 *
1222 1223 * - if this is a #db i.e. T_SGLSTP, %esi is live
1223 1224 * and contains the value of %db6
1224 1225 */
1225 1226
1226 1227 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1227 1228 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1228 1229 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1229 1230
1230 1231 /*
1231 1232 * We must first check if DTrace has set its NOFAULT bit. This
1232 1233 * regrettably must happen before the trap stack is recorded, because
1233 1234 * this requires a call to getpcstack() and may induce recursion if an
1234 1235 * fbt::getpcstack: enabling is inducing the bad load.
1235 1236 */
1236 1237 movl %gs:CPU_ID, %eax
1237 1238 shll $CPU_CORE_SHIFT, %eax
1238 1239 addl $cpu_core, %eax
1239 1240 movw CPUC_DTRACE_FLAGS(%eax), %cx
1240 1241 testw $CPU_DTRACE_NOFAULT, %cx
1241 1242 jnz .dtrace_induced
1242 1243
1243 1244 TRACE_STACK(%edi)
1244 1245
1245 1246 pushl %gs:CPU_ID
1246 1247 pushl %esi /* fault address for PGFLTs */
1247 1248 pushl %ebp /* ®s */
1248 1249
1249 1250 /*
1250 1251 * We know that this isn't a DTrace non-faulting load; we can now safely
1251 1252 * reenable interrupts. (In the case of pagefaults, we enter through an
1252 1253 * interrupt gate.)
1253 1254 */
1254 1255 ENABLE_INTR_FLAGS
1255 1256
1256 1257 call trap /* trap(rp, addr, cpuid) handles all traps */
1257 1258 addl $12, %esp /* get argument off stack */
1258 1259 jmp _sys_rtt
1259 1260
1260 1261 .dtrace_induced:
1261 1262 cmpw $KCS_SEL, REGOFF_CS(%ebp) /* test CS for user-mode trap */
1262 1263 jne 3f /* if from user, panic */
1263 1264
1264 1265 cmpl $T_PGFLT, REGOFF_TRAPNO(%ebp)
1265 1266 je 1f
1266 1267
1267 1268 cmpl $T_GPFLT, REGOFF_TRAPNO(%ebp)
1268 1269 je 0f
1269 1270
1270 1271 cmpl $T_ZERODIV, REGOFF_TRAPNO(%ebp)
1271 1272 jne 4f /* if not PF/GP/UD/DE, panic */
1272 1273
1273 1274 orw $CPU_DTRACE_DIVZERO, %cx
1274 1275 movw %cx, CPUC_DTRACE_FLAGS(%eax)
1275 1276 jmp 2f
1276 1277
1277 1278 0:
1278 1279 /*
1279 1280 * If we've taken a GPF, we don't (unfortunately) have the address that
1280 1281 * induced the fault. So instead of setting the fault to BADADDR,
1281 1282 * we'll set the fault to ILLOP.
1282 1283 */
1283 1284 orw $CPU_DTRACE_ILLOP, %cx
1284 1285 movw %cx, CPUC_DTRACE_FLAGS(%eax)
1285 1286 jmp 2f
1286 1287 1:
1287 1288 orw $CPU_DTRACE_BADADDR, %cx
1288 1289 movw %cx, CPUC_DTRACE_FLAGS(%eax) /* set fault to bad addr */
1289 1290 movl %esi, CPUC_DTRACE_ILLVAL(%eax)
1290 1291 /* fault addr is illegal value */
1291 1292 2:
1292 1293 pushl REGOFF_EIP(%ebp)
1293 1294 call dtrace_instr_size
1294 1295 addl $4, %esp
1295 1296 movl REGOFF_EIP(%ebp), %ecx
1296 1297 addl %eax, %ecx
1297 1298 movl %ecx, REGOFF_EIP(%ebp)
1298 1299 INTR_POP_KERNEL
1299 1300 IRET
1300 1301 /*NOTREACHED*/
1301 1302 3:
1302 1303 pushl $dtrace_badflags
1303 1304 call panic
1304 1305 4:
1305 1306 pushl $dtrace_badtrap
1306 1307 call panic
1307 1308 SET_SIZE(cmntrap)
1308 1309 SET_SIZE(_cmntrap)
1309 1310
1310 1311 #endif /* __i386 */
1311 1312
1312 1313 /*
1313 1314 * Declare a uintptr_t which has the size of _cmntrap to enable stack
1314 1315 * traceback code to know when a regs structure is on the stack.
1315 1316 */
1316 1317 .globl _cmntrap_size
1317 1318 .align CLONGSIZE
1318 1319 _cmntrap_size:
1319 1320 .NWORD . - _cmntrap
1320 1321 .type _cmntrap_size, @object
1321 1322
1322 1323 dtrace_badflags:
1323 1324 .string "bad DTrace flags"
1324 1325
1325 1326 dtrace_badtrap:
1326 1327 .string "bad DTrace trap"
1327 1328
1328 1329 #endif /* __lint */
1329 1330
1330 1331 #if defined(__lint)
1331 1332
1332 1333 /* ARGSUSED */
1333 1334 void
1334 1335 cmninttrap()
1335 1336 {}
1336 1337
1337 1338 #if !defined(__xpv)
1338 1339 void
1339 1340 bop_trap_handler(void)
1340 1341 {}
1341 1342 #endif
1342 1343
1343 1344 #else /* __lint */
1344 1345
1345 1346 .globl trap /* C handler called below */
1346 1347
1347 1348 #if defined(__amd64)
1348 1349
1349 1350 ENTRY_NP(cmninttrap)
1350 1351
1351 1352 INTR_PUSH
1352 1353 INTGATE_INIT_KERNEL_FLAGS
1353 1354
1354 1355 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1355 1356 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1356 1357 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1357 1358
1358 1359 movq %rsp, %rbp
1359 1360
1360 1361 movl %gs:CPU_ID, %edx
1361 1362 xorl %esi, %esi
1362 1363 movq %rsp, %rdi
1363 1364 call trap /* trap(rp, addr, cpuid) handles all traps */
1364 1365 jmp _sys_rtt
1365 1366 SET_SIZE(cmninttrap)
1366 1367
1367 1368 #if !defined(__xpv)
1368 1369 /*
1369 1370 * Handle traps early in boot. Just revectors into C quickly as
1370 1371 * these are always fatal errors.
1371 1372 *
1372 1373 * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap().
1373 1374 */
1374 1375 ENTRY(bop_trap_handler)
1375 1376 movq %rsp, %rdi
1376 1377 sub $8, %rsp
1377 1378 call bop_trap
1378 1379 SET_SIZE(bop_trap_handler)
1379 1380 #endif
1380 1381
1381 1382 #elif defined(__i386)
1382 1383
1383 1384 ENTRY_NP(cmninttrap)
1384 1385
1385 1386 INTR_PUSH
1386 1387 INTGATE_INIT_KERNEL_FLAGS
1387 1388
1388 1389 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1389 1390 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1390 1391 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1391 1392
1392 1393 movl %esp, %ebp
1393 1394
1394 1395 TRACE_STACK(%edi)
1395 1396
1396 1397 pushl %gs:CPU_ID
1397 1398 pushl $0
1398 1399 pushl %ebp
1399 1400 call trap /* trap(rp, addr, cpuid) handles all traps */
1400 1401 addl $12, %esp
1401 1402 jmp _sys_rtt
1402 1403 SET_SIZE(cmninttrap)
1403 1404
1404 1405 #if !defined(__xpv)
1405 1406 /*
1406 1407 * Handle traps early in boot. Just revectors into C quickly as
1407 1408 * these are always fatal errors.
1408 1409 */
1409 1410 ENTRY(bop_trap_handler)
1410 1411 movl %esp, %eax
1411 1412 pushl %eax
1412 1413 call bop_trap
1413 1414 SET_SIZE(bop_trap_handler)
1414 1415 #endif
1415 1416
1416 1417 #endif /* __i386 */
1417 1418
1418 1419 #endif /* __lint */
1419 1420
1420 1421 #if defined(__lint)
1421 1422
1422 1423 /* ARGSUSED */
1423 1424 void
1424 1425 dtrace_trap()
1425 1426 {}
1426 1427
1427 1428 #else /* __lint */
1428 1429
1429 1430 .globl dtrace_user_probe
1430 1431
1431 1432 #if defined(__amd64)
1432 1433
1433 1434 ENTRY_NP(dtrace_trap)
1434 1435
1435 1436 INTR_PUSH
1436 1437
1437 1438 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
1438 1439 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) /* Uses label 9 */
1439 1440 TRACE_STAMP(%rdi) /* Clobbers %eax, %edx, uses 9 */
1440 1441
1441 1442 movq %rsp, %rbp
1442 1443
1443 1444 movl %gs:CPU_ID, %edx
1444 1445 #if defined(__xpv)
1445 1446 movq %gs:CPU_VCPU_INFO, %rsi
1446 1447 movq VCPU_INFO_ARCH_CR2(%rsi), %rsi
1447 1448 #else
1448 1449 movq %cr2, %rsi
1449 1450 #endif
1450 1451 movq %rsp, %rdi
1451 1452
1452 1453 ENABLE_INTR_FLAGS
1453 1454
1454 1455 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
1455 1456 jmp _sys_rtt
1456 1457
1457 1458 SET_SIZE(dtrace_trap)
1458 1459
1459 1460 #elif defined(__i386)
1460 1461
1461 1462 ENTRY_NP(dtrace_trap)
1462 1463
1463 1464 INTR_PUSH
1464 1465
1465 1466 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) /* Uses labels 8 and 9 */
1466 1467 TRACE_REGS(%edi, %esp, %ebx, %ecx) /* Uses label 9 */
1467 1468 TRACE_STAMP(%edi) /* Clobbers %eax, %edx, uses 9 */
1468 1469
1469 1470 movl %esp, %ebp
1470 1471
1471 1472 pushl %gs:CPU_ID
1472 1473 #if defined(__xpv)
1473 1474 movl %gs:CPU_VCPU_INFO, %eax
1474 1475 movl VCPU_INFO_ARCH_CR2(%eax), %eax
1475 1476 #else
1476 1477 movl %cr2, %eax
1477 1478 #endif
1478 1479 pushl %eax
1479 1480 pushl %ebp
1480 1481
1481 1482 ENABLE_INTR_FLAGS
1482 1483
1483 1484 call dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
1484 1485 addl $12, %esp /* get argument off stack */
1485 1486
1486 1487 jmp _sys_rtt
1487 1488 SET_SIZE(dtrace_trap)
1488 1489
1489 1490 #endif /* __i386 */
1490 1491
1491 1492 #endif /* __lint */
1492 1493
1493 1494 /*
1494 1495 * Return from _sys_trap routine.
1495 1496 */
1496 1497
1497 1498 #if defined(__lint)
1498 1499
1499 1500 void
1500 1501 lwp_rtt_initial(void)
1501 1502 {}
1502 1503
1503 1504 void
1504 1505 lwp_rtt(void)
1505 1506 {}
1506 1507
1507 1508 void
1508 1509 _sys_rtt(void)
1509 1510 {}
1510 1511
1511 1512 #else /* __lint */
1512 1513
1513 1514 ENTRY_NP(lwp_rtt_initial)
1514 1515 movq %gs:CPU_THREAD, %r15
1515 1516 movq T_STACK(%r15), %rsp /* switch to the thread stack */
1516 1517 movq %rsp, %rbp
1517 1518 call __dtrace_probe___proc_start
1518 1519 jmp _lwp_rtt
1519 1520
1520 1521 ENTRY_NP(lwp_rtt)
1521 1522
1522 1523 /*
1523 1524 * r14 lwp
1524 1525 * rdx lwp->lwp_procp
1525 1526 * r15 curthread
1526 1527 */
1527 1528
1528 1529 movq %gs:CPU_THREAD, %r15
1529 1530 movq T_STACK(%r15), %rsp /* switch to the thread stack */
1530 1531 movq %rsp, %rbp
1531 1532 _lwp_rtt:
1532 1533 call __dtrace_probe___proc_lwp__start
1533 1534 movq %gs:CPU_LWP, %r14
1534 1535 movq LWP_PROCP(%r14), %rdx
1535 1536
1536 1537 /*
1537 1538 * XX64 Is the stack misaligned correctly at this point?
1538 1539 * If not, we need to do a push before calling anything ..
1539 1540 */
1540 1541
1541 1542 #if defined(DEBUG)
1542 1543 /*
1543 1544 * If we were to run lwp_savectx at this point -without-
1544 1545 * pcb_rupdate being set to 1, we'd end up sampling the hardware
1545 1546 * state left by the previous running lwp, rather than setting
1546 1547 * the values requested by the lwp creator. Bad.
1547 1548 */
1548 1549 testb $0x1, PCB_RUPDATE(%r14)
1549 1550 jne 1f
1550 1551 leaq _no_pending_updates(%rip), %rdi
1551 1552 movl $__LINE__, %esi
1552 1553 movq %r14, %rdx
1553 1554 xorl %eax, %eax
1554 1555 call panic
1555 1556 1:
1556 1557 #endif
1557 1558
1558 1559 /*
1559 1560 * If agent lwp, clear %fs and %gs
1560 1561 */
1561 1562 cmpq %r15, P_AGENTTP(%rdx)
1562 1563 jne 1f
1563 1564 xorl %ecx, %ecx
1564 1565 movq %rcx, REGOFF_FS(%rsp)
1565 1566 movq %rcx, REGOFF_GS(%rsp)
1566 1567 movw %cx, LWP_PCB_FS(%r14)
1567 1568 movw %cx, LWP_PCB_GS(%r14)
1568 1569 1:
1569 1570 call dtrace_systrace_rtt
1570 1571 movq REGOFF_RDX(%rsp), %rsi
1571 1572 movq REGOFF_RAX(%rsp), %rdi
1572 1573 call post_syscall /* post_syscall(rval1, rval2) */
1573 1574
1574 1575 /*
1575 1576 * XXX - may want a fast path that avoids sys_rtt_common in the
1576 1577 * most common case.
1577 1578 */
1578 1579 ALTENTRY(_sys_rtt)
1579 1580 CLI(%rax) /* disable interrupts */
1580 1581 ALTENTRY(_sys_rtt_ints_disabled)
1581 1582 movq %rsp, %rdi /* pass rp to sys_rtt_common */
1582 1583 call sys_rtt_common /* do common sys_rtt tasks */
1583 1584 testq %rax, %rax /* returning to userland? */
1584 1585 jz sr_sup
1585 1586
1586 1587 /*
1587 1588 * Return to user
1588 1589 */
1589 1590 ASSERT_UPCALL_MASK_IS_SET
1590 1591 cmpw $UCS_SEL, REGOFF_CS(%rsp) /* test for native (64-bit) lwp? */
1591 1592 je sys_rtt_syscall
1592 1593
1593 1594 /*
1594 1595 * Return to 32-bit userland
1595 1596 */
1596 1597 ALTENTRY(sys_rtt_syscall32)
1597 1598 USER32_POP
1598 1599 jmp tr_iret_user
1599 1600 /*NOTREACHED*/
1600 1601
1601 1602 ALTENTRY(sys_rtt_syscall)
1602 1603 /*
1603 1604 * Return to 64-bit userland
1604 1605 */
1605 1606 USER_POP
1606 1607 ALTENTRY(nopop_sys_rtt_syscall)
1607 1608 jmp tr_iret_user
1608 1609 /*NOTREACHED*/
1609 1610 SET_SIZE(nopop_sys_rtt_syscall)
1610 1611
1611 1612 /*
1612 1613 * Return to supervisor
1613 1614 * NOTE: to make the check in trap() that tests if we are executing
1614 1615 * segment register fixup/restore code work properly, sr_sup MUST be
1615 1616 * after _sys_rtt .
1616 1617 */
1617 1618 ALTENTRY(sr_sup)
1618 1619 /*
1619 1620 * Restore regs before doing iretq to kernel mode
1620 1621 */
1621 1622 INTR_POP
1622 1623 jmp tr_iret_kernel
1623 1624 .globl _sys_rtt_end
1624 1625 _sys_rtt_end:
1625 1626 /*NOTREACHED*/
1626 1627 SET_SIZE(sr_sup)
1627 1628 SET_SIZE(_sys_rtt_end)
1628 1629 SET_SIZE(lwp_rtt)
1629 1630 SET_SIZE(lwp_rtt_initial)
1630 1631 SET_SIZE(_sys_rtt_ints_disabled)
1631 1632 SET_SIZE(_sys_rtt)
1632 1633 SET_SIZE(sys_rtt_syscall)
1633 1634 SET_SIZE(sys_rtt_syscall32)
1634 1635
1635 1636 #endif /* __lint */
1636 1637
1637 1638 #if defined(__lint)
1638 1639
1639 1640 /*
1640 1641 * So why do we have to deal with all this crud in the world of ia32?
1641 1642 *
1642 1643 * Basically there are four classes of ia32 implementations, those that do not
1643 1644 * have a TSC, those that have a marginal TSC that is broken to the extent
1644 1645 * that it is useless, those that have a marginal TSC that is not quite so
1645 1646 * horribly broken and can be used with some care, and those that have a
1646 1647 * reliable TSC. This crud has to be here in order to sift through all the
1647 1648 * variants.
1648 1649 */
1649 1650
1650 1651 /*ARGSUSED*/
1651 1652 uint64_t
1652 1653 freq_tsc(uint32_t *pit_counter)
1653 1654 {
1654 1655 return (0);
1655 1656 }
1656 1657
1657 1658 #else /* __lint */
1658 1659
1659 1660 #if defined(__amd64)
1660 1661
1661 1662 /*
1662 1663 * XX64 quick and dirty port from the i386 version. Since we
1663 1664 * believe the amd64 tsc is more reliable, could this code be
1664 1665 * simpler?
1665 1666 */
1666 1667 ENTRY_NP(freq_tsc)
1667 1668 pushq %rbp
1668 1669 movq %rsp, %rbp
1669 1670 movq %rdi, %r9 /* save pit_counter */
1670 1671 pushq %rbx
1671 1672
1672 1673 / We have a TSC, but we have no way in general to know how reliable it is.
1673 1674 / Usually a marginal TSC behaves appropriately unless not enough time
1674 1675 / elapses between reads. A reliable TSC can be read as often and as rapidly
1675 1676 / as desired. The simplistic approach of reading the TSC counter and
1676 1677 / correlating to the PIT counter cannot be naively followed. Instead estimates
1677 1678 / have to be taken to successively refine a guess at the speed of the cpu
1678 1679 / and then the TSC and PIT counter are correlated. In practice very rarely
1679 1680 / is more than one quick loop required for an estimate. Measures have to be
1680 1681 / taken to prevent the PIT counter from wrapping beyond its resolution and for
1681 1682 / measuring the clock rate of very fast processors.
1682 1683 /
1683 1684 / The following constant can be tuned. It should be such that the loop does
1684 1685 / not take too many nor too few PIT counts to execute. If this value is too
1685 1686 / large, then on slow machines the loop will take a long time, or the PIT
1686 1687 / counter may even wrap. If this value is too small, then on fast machines
1687 1688 / the PIT counter may count so few ticks that the resolution of the PIT
1688 1689 / itself causes a bad guess. Because this code is used in machines with
1689 1690 / marginal TSC's and/or IO, if this value is too small on those, it may
1690 1691 / cause the calculated cpu frequency to vary slightly from boot to boot.
1691 1692 /
1692 1693 / In all cases even if this constant is set inappropriately, the algorithm
1693 1694 / will still work and the caller should be able to handle variances in the
1694 1695 / calculation of cpu frequency, but the calculation will be inefficient and
1695 1696 / take a disproportionate amount of time relative to a well selected value.
1696 1697 / As the slowest supported cpu becomes faster, this constant should be
1697 1698 / carefully increased.
1698 1699
1699 1700 movl $0x8000, %ecx
1700 1701
1701 1702 / to make sure the instruction cache has been warmed
1702 1703 clc
1703 1704
1704 1705 jmp freq_tsc_loop
1705 1706
1706 1707 / The following block of code up to and including the latching of the PIT
1707 1708 / counter after freq_tsc_perf_loop is very critical and very carefully
1708 1709 / written, it should only be modified with great care. freq_tsc_loop to
1709 1710 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
1710 1711 / freq_tsc_perf_loop up to the unlatching of the PIT counter.
1711 1712
1712 1713 .align 32
1713 1714 freq_tsc_loop:
1714 1715 / save the loop count in %ebx
1715 1716 movl %ecx, %ebx
1716 1717
1717 1718 / initialize the PIT counter and start a count down
1718 1719 movb $PIT_LOADMODE, %al
1719 1720 outb $PITCTL_PORT
1720 1721 movb $0xff, %al
1721 1722 outb $PITCTR0_PORT
1722 1723 outb $PITCTR0_PORT
1723 1724
1724 1725 / read the TSC and store the TS in %edi:%esi
1725 1726 rdtsc
1726 1727 movl %eax, %esi
1727 1728
1728 1729 freq_tsc_perf_loop:
1729 1730 movl %edx, %edi
1730 1731 movl %eax, %esi
1731 1732 movl %edx, %edi
1732 1733 loop freq_tsc_perf_loop
1733 1734
1734 1735 / read the TSC and store the LSW in %ecx
1735 1736 rdtsc
1736 1737 movl %eax, %ecx
1737 1738
1738 1739 / latch the PIT counter and status
1739 1740 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
1740 1741 outb $PITCTL_PORT
1741 1742
1742 1743 / remember if the icache has been warmed
1743 1744 setc %ah
1744 1745
1745 1746 / read the PIT status
1746 1747 inb $PITCTR0_PORT
1747 1748 shll $8, %eax
1748 1749
1749 1750 / read PIT count
1750 1751 inb $PITCTR0_PORT
1751 1752 shll $8, %eax
1752 1753 inb $PITCTR0_PORT
1753 1754 bswap %eax
1754 1755
1755 1756 / check to see if the PIT count was loaded into the CE
1756 1757 btw $_CONST(PITSTAT_NULLCNT+8), %ax
1757 1758 jc freq_tsc_increase_count
1758 1759
1759 1760 / check to see if PIT counter wrapped
1760 1761 btw $_CONST(PITSTAT_OUTPUT+8), %ax
1761 1762 jnc freq_tsc_pit_did_not_wrap
1762 1763
1763 1764 / halve count
1764 1765 shrl $1, %ebx
1765 1766 movl %ebx, %ecx
1766 1767
1767 1768 / the instruction cache has been warmed
1768 1769 stc
1769 1770
1770 1771 jmp freq_tsc_loop
1771 1772
1772 1773 freq_tsc_increase_count:
1773 1774 shll $1, %ebx
1774 1775 jc freq_tsc_too_fast
1775 1776
1776 1777 movl %ebx, %ecx
1777 1778
1778 1779 / the instruction cache has been warmed
1779 1780 stc
1780 1781
1781 1782 jmp freq_tsc_loop
1782 1783
1783 1784 freq_tsc_pit_did_not_wrap:
1784 1785 roll $16, %eax
1785 1786
1786 1787 cmpw $0x2000, %ax
1787 1788 notw %ax
1788 1789 jb freq_tsc_sufficient_duration
1789 1790
1790 1791 freq_tsc_calculate:
1791 1792 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
1792 1793 / then on the second CLK pulse the CE is decremented, therefore mode 0
1793 1794 / is really a (count + 1) counter, ugh
1794 1795 xorl %esi, %esi
1795 1796 movw %ax, %si
1796 1797 incl %esi
1797 1798
1798 1799 movl $0xf000, %eax
1799 1800 mull %ebx
1800 1801
1801 1802 / tuck away (target_pit_count * loop_count)
1802 1803 movl %edx, %ecx
1803 1804 movl %eax, %ebx
1804 1805
1805 1806 movl %esi, %eax
1806 1807 movl $0xffffffff, %edx
1807 1808 mull %edx
1808 1809
1809 1810 addl %esi, %eax
1810 1811 adcl $0, %edx
1811 1812
1812 1813 cmpl %ecx, %edx
1813 1814 ja freq_tsc_div_safe
1814 1815 jb freq_tsc_too_fast
1815 1816
1816 1817 cmpl %ebx, %eax
1817 1818 jbe freq_tsc_too_fast
1818 1819
1819 1820 freq_tsc_div_safe:
1820 1821 movl %ecx, %edx
1821 1822 movl %ebx, %eax
1822 1823
1823 1824 movl %esi, %ecx
1824 1825 divl %ecx
1825 1826
1826 1827 movl %eax, %ecx
1827 1828
1828 1829 / the instruction cache has been warmed
1829 1830 stc
1830 1831
1831 1832 jmp freq_tsc_loop
1832 1833
1833 1834 freq_tsc_sufficient_duration:
1834 1835 / test to see if the icache has been warmed
1835 1836 btl $16, %eax
1836 1837 jnc freq_tsc_calculate
1837 1838
1838 1839 / recall mode 0 is a (count + 1) counter
1839 1840 andl $0xffff, %eax
1840 1841 incl %eax
1841 1842
1842 1843 / save the number of PIT counts
1843 1844 movl %eax, (%r9)
1844 1845
1845 1846 / calculate the number of TS's that elapsed
1846 1847 movl %ecx, %eax
1847 1848 subl %esi, %eax
1848 1849 sbbl %edi, %edx
1849 1850
1850 1851 jmp freq_tsc_end
1851 1852
1852 1853 freq_tsc_too_fast:
1853 1854 / return 0 as a 64 bit quantity
1854 1855 xorl %eax, %eax
1855 1856 xorl %edx, %edx
1856 1857
1857 1858 freq_tsc_end:
1858 1859 shlq $32, %rdx
1859 1860 orq %rdx, %rax
1860 1861
1861 1862 popq %rbx
1862 1863 leaveq
1863 1864 ret
1864 1865 SET_SIZE(freq_tsc)
1865 1866
1866 1867 #elif defined(__i386)
1867 1868
1868 1869 ENTRY_NP(freq_tsc)
1869 1870 pushl %ebp
1870 1871 movl %esp, %ebp
1871 1872 pushl %edi
1872 1873 pushl %esi
1873 1874 pushl %ebx
1874 1875
1875 1876 / We have a TSC, but we have no way in general to know how reliable it is.
1876 1877 / Usually a marginal TSC behaves appropriately unless not enough time
1877 1878 / elapses between reads. A reliable TSC can be read as often and as rapidly
1878 1879 / as desired. The simplistic approach of reading the TSC counter and
1879 1880 / correlating to the PIT counter cannot be naively followed. Instead estimates
1880 1881 / have to be taken to successively refine a guess at the speed of the cpu
1881 1882 / and then the TSC and PIT counter are correlated. In practice very rarely
1882 1883 / is more than one quick loop required for an estimate. Measures have to be
1883 1884 / taken to prevent the PIT counter from wrapping beyond its resolution and for
1884 1885 / measuring the clock rate of very fast processors.
1885 1886 /
1886 1887 / The following constant can be tuned. It should be such that the loop does
1887 1888 / not take too many nor too few PIT counts to execute. If this value is too
1888 1889 / large, then on slow machines the loop will take a long time, or the PIT
1889 1890 / counter may even wrap. If this value is too small, then on fast machines
1890 1891 / the PIT counter may count so few ticks that the resolution of the PIT
1891 1892 / itself causes a bad guess. Because this code is used in machines with
1892 1893 / marginal TSC's and/or IO, if this value is too small on those, it may
1893 1894 / cause the calculated cpu frequency to vary slightly from boot to boot.
1894 1895 /
1895 1896 / In all cases even if this constant is set inappropriately, the algorithm
1896 1897 / will still work and the caller should be able to handle variances in the
1897 1898 / calculation of cpu frequency, but the calculation will be inefficient and
1898 1899 / take a disproportionate amount of time relative to a well selected value.
1899 1900 / As the slowest supported cpu becomes faster, this constant should be
1900 1901 / carefully increased.
1901 1902
1902 1903 movl $0x8000, %ecx
1903 1904
1904 1905 / to make sure the instruction cache has been warmed
1905 1906 clc
1906 1907
1907 1908 jmp freq_tsc_loop
1908 1909
1909 1910 / The following block of code up to and including the latching of the PIT
1910 1911 / counter after freq_tsc_perf_loop is very critical and very carefully
1911 1912 / written, it should only be modified with great care. freq_tsc_loop to
1912 1913 / freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
1913 1914 / freq_tsc_perf_loop up to the unlatching of the PIT counter.
1914 1915
1915 1916 .align 32
1916 1917 freq_tsc_loop:
1917 1918 / save the loop count in %ebx
1918 1919 movl %ecx, %ebx
1919 1920
1920 1921 / initialize the PIT counter and start a count down
1921 1922 movb $PIT_LOADMODE, %al
1922 1923 outb $PITCTL_PORT
1923 1924 movb $0xff, %al
1924 1925 outb $PITCTR0_PORT
1925 1926 outb $PITCTR0_PORT
1926 1927
1927 1928 / read the TSC and store the TS in %edi:%esi
1928 1929 rdtsc
1929 1930 movl %eax, %esi
1930 1931
1931 1932 freq_tsc_perf_loop:
1932 1933 movl %edx, %edi
1933 1934 movl %eax, %esi
1934 1935 movl %edx, %edi
1935 1936 loop freq_tsc_perf_loop
1936 1937
1937 1938 / read the TSC and store the LSW in %ecx
1938 1939 rdtsc
1939 1940 movl %eax, %ecx
1940 1941
1941 1942 / latch the PIT counter and status
1942 1943 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
1943 1944 outb $PITCTL_PORT
1944 1945
1945 1946 / remember if the icache has been warmed
1946 1947 setc %ah
1947 1948
1948 1949 / read the PIT status
1949 1950 inb $PITCTR0_PORT
1950 1951 shll $8, %eax
1951 1952
1952 1953 / read PIT count
1953 1954 inb $PITCTR0_PORT
1954 1955 shll $8, %eax
1955 1956 inb $PITCTR0_PORT
1956 1957 bswap %eax
1957 1958
1958 1959 / check to see if the PIT count was loaded into the CE
1959 1960 btw $_CONST(PITSTAT_NULLCNT+8), %ax
1960 1961 jc freq_tsc_increase_count
1961 1962
1962 1963 / check to see if PIT counter wrapped
1963 1964 btw $_CONST(PITSTAT_OUTPUT+8), %ax
1964 1965 jnc freq_tsc_pit_did_not_wrap
1965 1966
1966 1967 / halve count
1967 1968 shrl $1, %ebx
1968 1969 movl %ebx, %ecx
1969 1970
1970 1971 / the instruction cache has been warmed
1971 1972 stc
1972 1973
1973 1974 jmp freq_tsc_loop
1974 1975
1975 1976 freq_tsc_increase_count:
1976 1977 shll $1, %ebx
1977 1978 jc freq_tsc_too_fast
1978 1979
1979 1980 movl %ebx, %ecx
1980 1981
1981 1982 / the instruction cache has been warmed
1982 1983 stc
1983 1984
1984 1985 jmp freq_tsc_loop
1985 1986
1986 1987 freq_tsc_pit_did_not_wrap:
1987 1988 roll $16, %eax
1988 1989
1989 1990 cmpw $0x2000, %ax
1990 1991 notw %ax
1991 1992 jb freq_tsc_sufficient_duration
1992 1993
1993 1994 freq_tsc_calculate:
1994 1995 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
1995 1996 / then on the second CLK pulse the CE is decremented, therefore mode 0
1996 1997 / is really a (count + 1) counter, ugh
1997 1998 xorl %esi, %esi
1998 1999 movw %ax, %si
1999 2000 incl %esi
2000 2001
2001 2002 movl $0xf000, %eax
2002 2003 mull %ebx
2003 2004
2004 2005 / tuck away (target_pit_count * loop_count)
2005 2006 movl %edx, %ecx
2006 2007 movl %eax, %ebx
2007 2008
2008 2009 movl %esi, %eax
2009 2010 movl $0xffffffff, %edx
2010 2011 mull %edx
2011 2012
2012 2013 addl %esi, %eax
2013 2014 adcl $0, %edx
2014 2015
2015 2016 cmpl %ecx, %edx
2016 2017 ja freq_tsc_div_safe
2017 2018 jb freq_tsc_too_fast
2018 2019
2019 2020 cmpl %ebx, %eax
2020 2021 jbe freq_tsc_too_fast
2021 2022
2022 2023 freq_tsc_div_safe:
2023 2024 movl %ecx, %edx
2024 2025 movl %ebx, %eax
2025 2026
2026 2027 movl %esi, %ecx
2027 2028 divl %ecx
2028 2029
2029 2030 movl %eax, %ecx
2030 2031
2031 2032 / the instruction cache has been warmed
2032 2033 stc
2033 2034
2034 2035 jmp freq_tsc_loop
2035 2036
2036 2037 freq_tsc_sufficient_duration:
2037 2038 / test to see if the icache has been warmed
2038 2039 btl $16, %eax
2039 2040 jnc freq_tsc_calculate
2040 2041
2041 2042 / recall mode 0 is a (count + 1) counter
2042 2043 andl $0xffff, %eax
2043 2044 incl %eax
2044 2045
2045 2046 / save the number of PIT counts
2046 2047 movl 8(%ebp), %ebx
2047 2048 movl %eax, (%ebx)
2048 2049
2049 2050 / calculate the number of TS's that elapsed
2050 2051 movl %ecx, %eax
2051 2052 subl %esi, %eax
2052 2053 sbbl %edi, %edx
2053 2054
2054 2055 jmp freq_tsc_end
2055 2056
2056 2057 freq_tsc_too_fast:
2057 2058 / return 0 as a 64 bit quantity
2058 2059 xorl %eax, %eax
2059 2060 xorl %edx, %edx
2060 2061
2061 2062 freq_tsc_end:
2062 2063 popl %ebx
2063 2064 popl %esi
2064 2065 popl %edi
2065 2066 popl %ebp
2066 2067 ret
2067 2068 SET_SIZE(freq_tsc)
2068 2069
2069 2070 #endif /* __i386 */
2070 2071 #endif /* __lint */
2071 2072
2072 2073 #if !defined(__amd64)
2073 2074 #if defined(__lint)
2074 2075
2075 2076 /*
2076 2077 * We do not have a TSC so we use a block of instructions with well known
2077 2078 * timings.
2078 2079 */
2079 2080
2080 2081 /*ARGSUSED*/
2081 2082 uint64_t
2082 2083 freq_notsc(uint32_t *pit_counter)
2083 2084 {
2084 2085 return (0);
2085 2086 }
2086 2087
2087 2088 #else /* __lint */
2088 2089 ENTRY_NP(freq_notsc)
2089 2090 pushl %ebp
2090 2091 movl %esp, %ebp
2091 2092 pushl %edi
2092 2093 pushl %esi
2093 2094 pushl %ebx
2094 2095
2095 2096 / initial count for the idivl loop
2096 2097 movl $0x1000, %ecx
2097 2098
2098 2099 / load the divisor
2099 2100 movl $1, %ebx
2100 2101
2101 2102 jmp freq_notsc_loop
2102 2103
2103 2104 .align 16
2104 2105 freq_notsc_loop:
2105 2106 / set high 32 bits of dividend to zero
2106 2107 xorl %edx, %edx
2107 2108
2108 2109 / save the loop count in %edi
2109 2110 movl %ecx, %edi
2110 2111
2111 2112 / initialize the PIT counter and start a count down
2112 2113 movb $PIT_LOADMODE, %al
2113 2114 outb $PITCTL_PORT
2114 2115 movb $0xff, %al
2115 2116 outb $PITCTR0_PORT
2116 2117 outb $PITCTR0_PORT
2117 2118
2118 2119 / set low 32 bits of dividend to zero
2119 2120 xorl %eax, %eax
2120 2121
2121 2122 / It is vital that the arguments to idivl be set appropriately because on some
2122 2123 / cpu's this instruction takes more or less clock ticks depending on its
2123 2124 / arguments.
2124 2125 freq_notsc_perf_loop:
2125 2126 idivl %ebx
2126 2127 idivl %ebx
2127 2128 idivl %ebx
2128 2129 idivl %ebx
2129 2130 idivl %ebx
2130 2131 loop freq_notsc_perf_loop
2131 2132
2132 2133 / latch the PIT counter and status
2133 2134 movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
2134 2135 outb $PITCTL_PORT
2135 2136
2136 2137 / read the PIT status
2137 2138 inb $PITCTR0_PORT
2138 2139 shll $8, %eax
2139 2140
2140 2141 / read PIT count
2141 2142 inb $PITCTR0_PORT
2142 2143 shll $8, %eax
2143 2144 inb $PITCTR0_PORT
2144 2145 bswap %eax
2145 2146
2146 2147 / check to see if the PIT count was loaded into the CE
2147 2148 btw $_CONST(PITSTAT_NULLCNT+8), %ax
2148 2149 jc freq_notsc_increase_count
2149 2150
2150 2151 / check to see if PIT counter wrapped
2151 2152 btw $_CONST(PITSTAT_OUTPUT+8), %ax
2152 2153 jnc freq_notsc_pit_did_not_wrap
2153 2154
2154 2155 / halve count
2155 2156 shrl $1, %edi
2156 2157 movl %edi, %ecx
2157 2158
2158 2159 jmp freq_notsc_loop
2159 2160
2160 2161 freq_notsc_increase_count:
2161 2162 shll $1, %edi
2162 2163 jc freq_notsc_too_fast
2163 2164
2164 2165 movl %edi, %ecx
2165 2166
2166 2167 jmp freq_notsc_loop
2167 2168
2168 2169 freq_notsc_pit_did_not_wrap:
2169 2170 shrl $16, %eax
2170 2171
2171 2172 cmpw $0x2000, %ax
2172 2173 notw %ax
2173 2174 jb freq_notsc_sufficient_duration
2174 2175
2175 2176 freq_notsc_calculate:
2176 2177 / in mode 0, the PIT loads the count into the CE on the first CLK pulse,
2177 2178 / then on the second CLK pulse the CE is decremented, therefore mode 0
2178 2179 / is really a (count + 1) counter, ugh
2179 2180 xorl %esi, %esi
2180 2181 movw %ax, %si
2181 2182 incl %esi
2182 2183
2183 2184 movl %edi, %eax
2184 2185 movl $0xf000, %ecx
2185 2186 mull %ecx
2186 2187
2187 2188 / tuck away (target_pit_count * loop_count)
2188 2189 movl %edx, %edi
2189 2190 movl %eax, %ecx
2190 2191
2191 2192 movl %esi, %eax
2192 2193 movl $0xffffffff, %edx
2193 2194 mull %edx
2194 2195
2195 2196 addl %esi, %eax
2196 2197 adcl $0, %edx
2197 2198
2198 2199 cmpl %edi, %edx
2199 2200 ja freq_notsc_div_safe
2200 2201 jb freq_notsc_too_fast
2201 2202
2202 2203 cmpl %ecx, %eax
2203 2204 jbe freq_notsc_too_fast
2204 2205
2205 2206 freq_notsc_div_safe:
2206 2207 movl %edi, %edx
2207 2208 movl %ecx, %eax
2208 2209
2209 2210 movl %esi, %ecx
2210 2211 divl %ecx
2211 2212
2212 2213 movl %eax, %ecx
2213 2214
2214 2215 jmp freq_notsc_loop
2215 2216
2216 2217 freq_notsc_sufficient_duration:
2217 2218 / recall mode 0 is a (count + 1) counter
2218 2219 incl %eax
2219 2220
2220 2221 / save the number of PIT counts
2221 2222 movl 8(%ebp), %ebx
2222 2223 movl %eax, (%ebx)
2223 2224
2224 2225 / calculate the number of cpu clock ticks that elapsed
2225 2226 cmpl $X86_VENDOR_Cyrix, x86_vendor
2226 2227 jz freq_notsc_notcyrix
2227 2228
2228 2229 / freq_notsc_perf_loop takes 86 clock cycles on Cyrix 6x86 cores
2229 2230 movl $86, %eax
2230 2231 jmp freq_notsc_calculate_tsc
2231 2232
2232 2233 freq_notsc_notcyrix:
2233 2234 / freq_notsc_perf_loop takes 237 clock cycles on Intel Pentiums
2234 2235 movl $237, %eax
2235 2236
2236 2237 freq_notsc_calculate_tsc:
2237 2238 mull %edi
2238 2239
2239 2240 jmp freq_notsc_end
2240 2241
2241 2242 freq_notsc_too_fast:
2242 2243 / return 0 as a 64 bit quantity
2243 2244 xorl %eax, %eax
2244 2245 xorl %edx, %edx
2245 2246
2246 2247 freq_notsc_end:
2247 2248 popl %ebx
2248 2249 popl %esi
2249 2250 popl %edi
2250 2251 popl %ebp
2251 2252
2252 2253 ret
2253 2254 SET_SIZE(freq_notsc)
2254 2255
2255 2256 #endif /* __lint */
2256 2257 #endif /* !__amd64 */
2257 2258
2258 2259 #if !defined(__lint)
2259 2260 .data
2260 2261 #if !defined(__amd64)
2261 2262 .align 4
2262 2263 cpu_vendor:
2263 2264 .long 0, 0, 0 /* Vendor ID string returned */
2264 2265
2265 2266 .globl CyrixInstead
2266 2267
2267 2268 .globl x86_featureset
2268 2269 .globl x86_type
2269 2270 .globl x86_vendor
2270 2271 #endif
2271 2272
2272 2273 #endif /* __lint */
|
↓ open down ↓ |
1059 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX