8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Assembly code support for the Cheetah+ module
26 */
27
28 #pragma ident "%Z%%M% %I% %E% SMI"
29
30 #if !defined(lint)
31 #include "assym.h"
32 #endif /* lint */
33
34 #include <sys/asm_linkage.h>
35 #include <sys/mmu.h>
36 #include <vm/hat_sfmmu.h>
37 #include <sys/machparam.h>
38 #include <sys/machcpuvar.h>
39 #include <sys/machthread.h>
40 #include <sys/machtrap.h>
41 #include <sys/privregs.h>
42 #include <sys/asm_linkage.h>
43 #include <sys/trap.h>
44 #include <sys/cheetahregs.h>
45 #include <sys/us3_module.h>
46 #include <sys/xc_impl.h>
47 #include <sys/intreg.h>
48 #include <sys/async.h>
49 #include <sys/clock.h>
50 #include <sys/cheetahasm.h>
51 #include <sys/cmpregs.h>
52
53 #ifdef TRAPTRACE
54 #include <sys/traptrace.h>
55 #endif /* TRAPTRACE */
56
57
58 #if !defined(lint)
59
60 /* BEGIN CSTYLED */
61
62 /*
63 * Cheetah+ version to reflush an Ecache line by index.
64 *
65 * By default we assume the Ecache is 2-way so we flush both
66 * ways. Even if the cache is direct-mapped no harm will come
67 * from performing the flush twice, apart from perhaps a performance
68 * penalty.
69 *
70 * XXX - scr2 not used.
71 */
72 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2) \
73 ldxa [index]ASI_EC_DIAG, %g0; \
74 ldxa [index + ec_set_size]ASI_EC_DIAG, %g0;
75
76 /*
77 * Cheetah+ version of ecache_flush_line. Uses Cheetah+ Ecache Displacement
78 * Flush feature.
79 */
130 * Output: Index into the L2$ of the line to be flushed.
131 * l3_idx_out Input: scratch register.
132 * Output: Index into the L3$ of the line to be flushed.
133 * scr3 scratch register.
134 * scr4 scratch register.
135 *
136 */
137 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \
138 set PN_L3_SET_SIZE, l2_idx_out; \
139 sub l2_idx_out, 1, l2_idx_out; \
140 and physaddr, l2_idx_out, l3_idx_out; \
141 set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \
142 or l2_idx_out, l3_idx_out, l3_idx_out; \
143 set PN_L2_SET_SIZE, l2_idx_out; \
144 sub l2_idx_out, 1, l2_idx_out; \
145 and physaddr, l2_idx_out, l2_idx_out; \
146 set PN_L2_IDX_DISP_FLUSH, scr3; \
147 or l2_idx_out, scr3, l2_idx_out; \
148 PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
149
150 #endif /* !lint */
151
152 /*
153 * Fast ECC error at TL>0 handler
154 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter
155 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
156 * For a complete description of the Fast ECC at TL>0 handling see the
157 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
158 * us3_common_asm.s
159 */
160 #if defined(lint)
161
162 void
163 fast_ecc_tl1_err(void)
164 {}
165
166 #else /* lint */
167
168 .section ".text"
169 .align 64
170 ENTRY_NP(fast_ecc_tl1_err)
171
172 /*
173 * This macro turns off the D$/I$ if they are on and saves their
174 * original state in ch_err_tl1_tmp, saves all the %g registers in the
175 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
176 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will
177 * point to the ch_err_tl1_data structure and the original D$/I$ state
178 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1
179 * will be available.
180 */
181 CH_ERR_TL1_FECC_ENTER;
182
183 /*
184 * Get the diagnostic logout data. %g4 must be initialized to
185 * current CEEN state, %g5 must point to logout structure in
186 * ch_err_tl1_data_t. %g3 will contain the nesting count upon
187 * return.
396 or %g3, %g4, %g3 ! %g3 = original + current AFSR_EXT
397 btst C_AFSR_L3_WDU, %g3 ! L3_WDU in original or current AFSR?
398 bnz %xcc, fecc_tl1_err ! panic (saw L3_WDU and UCU or L3_UCU)
399 nop
400 6:
401 /*
402 * We fall into this macro if we've successfully logged the error in
403 * the ch_err_tl1_data structure and want the PIL15 softint to pick
404 * it up and log it. %g1 must point to the ch_err_tl1_data structure.
405 * Restores the %g registers and issues retry.
406 */
407 CH_ERR_TL1_EXIT;
408
409 /*
410 * Establish panic exit label.
411 */
412 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
413
414 SET_SIZE(fast_ecc_tl1_err)
415
416 #endif /* lint */
417
418
419 #if defined(lint)
420 /*
421 * scrubphys - Pass in the aligned physical memory address
422 * that you want to scrub, along with the ecache set size.
423 *
424 * 1) Displacement flush the E$ line corresponding to %addr.
425 * The first ldxa guarantees that the %addr is no longer in
426 * M, O, or E (goes to I or S (if instruction fetch also happens).
427 * 2) "Write" the data using a CAS %addr,%g0,%g0.
428 * The casxa guarantees a transition from I to M or S to M.
429 * 3) Displacement flush the E$ line corresponding to %addr.
430 * The second ldxa pushes the M line out of the ecache, into the
431 * writeback buffers, on the way to memory.
432 * 4) The "membar #Sync" pushes the cache line out of the writeback
433 * buffers onto the bus, on the way to dram finally.
434 *
435 * This is a modified version of the algorithm suggested by Gary Lauterbach.
436 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
437 * as modified, but then we found out that for spitfire, if it misses in the
438 * E$ it will probably install as an M, but if it hits in the E$, then it
439 * will stay E, if the store doesn't happen. So the first displacement flush
440 * should ensure that the CAS will miss in the E$. Arrgh.
441 */
442 /* ARGSUSED */
443 void
444 scrubphys(uint64_t paddr, int ecache_set_size)
445 {}
446
447 #else /* lint */
448 ENTRY(scrubphys)
449 rdpr %pstate, %o4
450 andn %o4, PSTATE_IE | PSTATE_AM, %o5
451 wrpr %o5, %g0, %pstate ! clear IE, AM bits
452
453 GET_CPU_IMPL(%o5) ! Panther Ecache is flushed differently
454 cmp %o5, PANTHER_IMPL
455 bne scrubphys_1
456 nop
457 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5)
458 casxa [%o0]ASI_MEM, %g0, %g0
459 PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0)
460 b scrubphys_2
461 nop
462 scrubphys_1:
463 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
464 casxa [%o0]ASI_MEM, %g0, %g0
465 ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
466 scrubphys_2:
467 wrpr %g0, %o4, %pstate ! restore earlier pstate register value
468
469 retl
470 membar #Sync ! move the data out of the load buffer
471 SET_SIZE(scrubphys)
472
473 #endif /* lint */
474
475
476 #if defined(lint)
477 /*
478 * clearphys - Pass in the physical memory address of the checkblock
479 * that you want to push out, cleared with a recognizable pattern,
480 * from the ecache.
481 *
482 * To ensure that the ecc gets recalculated after the bad data is cleared,
483 * we must write out enough data to fill the w$ line (64 bytes). So we read
484 * in an entire ecache subblock's worth of data, and write it back out.
485 * Then we overwrite the 16 bytes of bad data with the pattern.
486 */
487 /* ARGSUSED */
488 void
489 clearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize)
490 {
491 }
492
493 #else /* lint */
494 ENTRY(clearphys)
495 /* turn off IE, AM bits */
496 rdpr %pstate, %o4
497 andn %o4, PSTATE_IE | PSTATE_AM, %o5
498 wrpr %o5, %g0, %pstate
499
500 /* turn off NCEEN */
501 ldxa [%g0]ASI_ESTATE_ERR, %o5
502 andn %o5, EN_REG_NCEEN, %o3
503 stxa %o3, [%g0]ASI_ESTATE_ERR
504 membar #Sync
505
506 /* align address passed with 64 bytes subblock size */
507 mov CH_ECACHE_SUBBLK_SIZE, %o2
508 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
509
510 /* move the good data into the W$ */
511 clearphys_1:
512 subcc %o2, 8, %o2
513 ldxa [%g1 + %o2]ASI_MEM, %g2
531 nop
532 clearphys_2:
533 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
534 casxa [%o0]ASI_MEM, %g0, %g0
535 ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
536 clearphys_3:
537 /* clear the AFSR */
538 ldxa [%g0]ASI_AFSR, %o1
539 stxa %o1, [%g0]ASI_AFSR
540 membar #Sync
541
542 /* turn NCEEN back on */
543 stxa %o5, [%g0]ASI_ESTATE_ERR
544 membar #Sync
545
546 /* return and re-enable IE and AM */
547 retl
548 wrpr %g0, %o4, %pstate
549 SET_SIZE(clearphys)
550
551 #endif /* lint */
552
553
554 #if defined(lint)
555 /*
556 * Cheetah+ Ecache displacement flush the specified line from the E$
557 *
558 * For Panther, this means flushing the specified line from both the
559 * L2 cache and L3 cache.
560 *
561 * Register usage:
562 * %o0 - 64 bit physical address for flushing
563 * %o1 - Ecache set size
564 */
565 /*ARGSUSED*/
566 void
567 ecache_flush_line(uint64_t flushaddr, int ec_set_size)
568 {
569 }
570 #else /* lint */
571 ENTRY(ecache_flush_line)
572
573 GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently
574 cmp %o3, PANTHER_IMPL
575 bne ecache_flush_line_1
576 nop
577
578 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4)
579 b ecache_flush_line_2
580 nop
581 ecache_flush_line_1:
582 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
583 ecache_flush_line_2:
584 retl
585 nop
586 SET_SIZE(ecache_flush_line)
587 #endif /* lint */
588
589 #if defined(lint)
590 void
591 set_afsr_ext(uint64_t afsr_ext)
592 {
593 afsr_ext = afsr_ext;
594 }
595 #else /* lint */
596
597 ENTRY(set_afsr_ext)
598 set ASI_AFSR_EXT_VA, %o1
599 stxa %o0, [%o1]ASI_AFSR ! afsr_ext reg
600 membar #Sync
601 retl
602 nop
603 SET_SIZE(set_afsr_ext)
604
605 #endif /* lint */
606
607
608 #if defined(lint)
609 /*
610 * The CPU jumps here from the MMU exception handler if an ITLB parity
611 * error is detected and we are running on Panther.
612 *
613 * In this routine we collect diagnostic information and write it to our
614 * logout structure (if possible) and clear all ITLB entries that may have
615 * caused our parity trap.
616 * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
617 * and log any error messages. As for parameters to cpu_tlb_parity_error, we
618 * send two:
619 *
620 * %g2 - Contains the VA whose lookup in the ITLB caused the parity error
621 * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct,
622 * regardless of whether or not we actually used the logout struct.
623 *
624 * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
625 * parameters to the data contained in the logout structure in order to
626 * determine whether the logout information is valid for this particular
627 * error or not.
628 */
629 void
630 itlb_parity_trap(void)
631 {}
632
633 #else /* lint */
634
635 ENTRY_NP(itlb_parity_trap)
636 /*
637 * Collect important information about the trap which will be
638 * used as a parameter to the TL0 handler.
639 */
640 wr %g0, ASI_IMMU, %asi
641 rdpr %tpc, %g2 ! VA that caused the IMMU trap
642 ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page size
643 set PN_ITLB_PGSZ_MASK, %g4
644 and %g3, %g4, %g3
645 ldxa [MMU_TAG_ACCESS]%asi, %g4
646 set TAGREAD_CTX_MASK, %g5
647 and %g4, %g5, %g4
648 or %g4, %g3, %g3 ! 'or' in the trap context and
649 mov 1, %g4 ! add the IMMU flag to complete
650 sllx %g4, PN_TLO_INFO_IMMU_SHIFT, %g4
651 or %g4, %g3, %g3 ! the tlo_info field for logout
652 stxa %g0,[MMU_SFSR]%asi ! clear the SFSR
653 membar #Sync
654
737 membar #Sync
738
739 sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a
740 flush %g6 ! flush after writing MMU regs
741
742 /*
743 * at this point:
744 * %g2 - contains the VA whose lookup caused the trap
745 * %g3 - contains the tlo_info field
746 *
747 * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
748 * already at PIL 15. */
749 set cpu_tlb_parity_error, %g1
750 rdpr %pil, %g4
751 cmp %g4, PIL_14
752 movl %icc, PIL_14, %g4
753 ba sys_trap
754 nop
755 SET_SIZE(itlb_parity_trap)
756
757 #endif /* lint */
758
759 #if defined(lint)
760 /*
761 * The CPU jumps here from the MMU exception handler if a DTLB parity
762 * error is detected and we are running on Panther.
763 *
764 * In this routine we collect diagnostic information and write it to our
765 * logout structure (if possible) and clear all DTLB entries that may have
766 * caused our parity trap.
767 * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
768 * and log any error messages. As for parameters to cpu_tlb_parity_error, we
769 * send two:
770 *
771 * %g2 - Contains the VA whose lookup in the DTLB caused the parity error
772 * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct,
773 * regardless of whether or not we actually used the logout struct.
774 *
775 * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
776 * parameters to the data contained in the logout structure in order to
777 * determine whether the logout information is valid for this particular
778 * error or not.
779 */
780 void
781 dtlb_parity_trap(void)
782 {}
783
784 #else /* lint */
785
786 ENTRY_NP(dtlb_parity_trap)
787 /*
788 * Collect important information about the trap which will be
789 * used as a parameter to the TL0 handler.
790 */
791 wr %g0, ASI_DMMU, %asi
792 ldxa [MMU_SFAR]%asi, %g2 ! VA that caused the IMMU trap
793 ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page sizes
794 set PN_DTLB_PGSZ_MASK, %g4
795 and %g3, %g4, %g3
796 ldxa [MMU_TAG_ACCESS]%asi, %g4
797 set TAGREAD_CTX_MASK, %g5 ! 'or' in the trap context
798 and %g4, %g5, %g4 ! to complete the tlo_info
799 or %g4, %g3, %g3 ! field for logout
800 stxa %g0,[MMU_SFSR]%asi ! clear the SFSR
801 membar #Sync
802
803 /*
804 * at this point:
805 * %g2 - contains the VA whose lookup caused the trap
950 sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a
951 flush %g6 ! flush after writing MMU regs
952
953 /*
954 * at this point:
955 * %g2 - contains the VA whose lookup caused the trap
956 * %g3 - contains the tlo_info field
957 *
958 * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
959 * already at PIL 15. We do this even for TL>1 traps since
960 * those will lead to a system panic.
961 */
962 set cpu_tlb_parity_error, %g1
963 rdpr %pil, %g4
964 cmp %g4, PIL_14
965 movl %icc, PIL_14, %g4
966 ba sys_trap
967 nop
968 SET_SIZE(dtlb_parity_trap)
969
970 #endif /* lint */
971
972
973 #if defined(lint)
974 /*
975 * Calculates the Panther TLB index based on a virtual address and page size
976 *
977 * Register usage:
978 * %o0 - virtual address whose index we want
979 * %o1 - Page Size of the TLB in question as encoded in the
980 * ASI_[D|I]MMU_TAG_ACCESS_EXT register.
981 */
982 uint64_t
983 pn_get_tlb_index(uint64_t va, uint64_t pg_sz)
984 {
985 return ((va + pg_sz)-(va + pg_sz));
986 }
987 #else /* lint */
988 ENTRY(pn_get_tlb_index)
989
990 PN_GET_TLB_INDEX(%o0, %o1)
991
992 retl
993 nop
994 SET_SIZE(pn_get_tlb_index)
995 #endif /* lint */
996
997
998 #if defined(lint)
999 /*
1000 * For Panther CPUs we need to flush the IPB after any I$ or D$
1001 * parity errors are detected.
1002 */
1003 void
1004 flush_ipb(void)
1005 { return; }
1006
1007 #else /* lint */
1008
1009 ENTRY(flush_ipb)
1010 clr %o0
1011
1012 flush_ipb_1:
1013 stxa %g0, [%o0]ASI_IPB_TAG
1014 membar #Sync
1015 cmp %o0, PN_IPB_TAG_ADDR_MAX
1016 blt flush_ipb_1
1017 add %o0, PN_IPB_TAG_ADDR_LINESIZE, %o0
1018
1019 sethi %hi(FLUSH_ADDR), %o0
1020 flush %o0
1021 retl
1022 nop
1023 SET_SIZE(flush_ipb)
1024
1025 #endif /* lint */
1026
1027
|
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Assembly code support for the Cheetah+ module
26 */
27
28 #include "assym.h"
29
30 #include <sys/asm_linkage.h>
31 #include <sys/mmu.h>
32 #include <vm/hat_sfmmu.h>
33 #include <sys/machparam.h>
34 #include <sys/machcpuvar.h>
35 #include <sys/machthread.h>
36 #include <sys/machtrap.h>
37 #include <sys/privregs.h>
38 #include <sys/asm_linkage.h>
39 #include <sys/trap.h>
40 #include <sys/cheetahregs.h>
41 #include <sys/us3_module.h>
42 #include <sys/xc_impl.h>
43 #include <sys/intreg.h>
44 #include <sys/async.h>
45 #include <sys/clock.h>
46 #include <sys/cheetahasm.h>
47 #include <sys/cmpregs.h>
48
49 #ifdef TRAPTRACE
50 #include <sys/traptrace.h>
51 #endif /* TRAPTRACE */
52
53
54 /* BEGIN CSTYLED */
55
56 /*
57 * Cheetah+ version to reflush an Ecache line by index.
58 *
59 * By default we assume the Ecache is 2-way so we flush both
60 * ways. Even if the cache is direct-mapped no harm will come
61 * from performing the flush twice, apart from perhaps a performance
62 * penalty.
63 *
64 * XXX - scr2 not used.
65 */
66 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2) \
67 ldxa [index]ASI_EC_DIAG, %g0; \
68 ldxa [index + ec_set_size]ASI_EC_DIAG, %g0;
69
70 /*
71 * Cheetah+ version of ecache_flush_line. Uses Cheetah+ Ecache Displacement
72 * Flush feature.
73 */
124 * Output: Index into the L2$ of the line to be flushed.
125 * l3_idx_out Input: scratch register.
126 * Output: Index into the L3$ of the line to be flushed.
127 * scr3 scratch register.
128 * scr4 scratch register.
129 *
130 */
131 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \
132 set PN_L3_SET_SIZE, l2_idx_out; \
133 sub l2_idx_out, 1, l2_idx_out; \
134 and physaddr, l2_idx_out, l3_idx_out; \
135 set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \
136 or l2_idx_out, l3_idx_out, l3_idx_out; \
137 set PN_L2_SET_SIZE, l2_idx_out; \
138 sub l2_idx_out, 1, l2_idx_out; \
139 and physaddr, l2_idx_out, l2_idx_out; \
140 set PN_L2_IDX_DISP_FLUSH, scr3; \
141 or l2_idx_out, scr3, l2_idx_out; \
142 PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
143
144 /*
145 * Fast ECC error at TL>0 handler
146 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter
147 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
148 * For a complete description of the Fast ECC at TL>0 handling see the
149 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
150 * us3_common_asm.s
151 */
152
153 .section ".text"
154 .align 64
155 ENTRY_NP(fast_ecc_tl1_err)
156
157 /*
158 * This macro turns off the D$/I$ if they are on and saves their
159 * original state in ch_err_tl1_tmp, saves all the %g registers in the
160 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
161 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will
162 * point to the ch_err_tl1_data structure and the original D$/I$ state
163 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1
164 * will be available.
165 */
166 CH_ERR_TL1_FECC_ENTER;
167
168 /*
169 * Get the diagnostic logout data. %g4 must be initialized to
170 * current CEEN state, %g5 must point to logout structure in
171 * ch_err_tl1_data_t. %g3 will contain the nesting count upon
172 * return.
381 or %g3, %g4, %g3 ! %g3 = original + current AFSR_EXT
382 btst C_AFSR_L3_WDU, %g3 ! L3_WDU in original or current AFSR?
383 bnz %xcc, fecc_tl1_err ! panic (saw L3_WDU and UCU or L3_UCU)
384 nop
385 6:
386 /*
387 * We fall into this macro if we've successfully logged the error in
388 * the ch_err_tl1_data structure and want the PIL15 softint to pick
389 * it up and log it. %g1 must point to the ch_err_tl1_data structure.
390 * Restores the %g registers and issues retry.
391 */
392 CH_ERR_TL1_EXIT;
393
394 /*
395 * Establish panic exit label.
396 */
397 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
398
399 SET_SIZE(fast_ecc_tl1_err)
400
401
402 /*
403 * scrubphys - Pass in the aligned physical memory address
404 * that you want to scrub, along with the ecache set size.
405 *
406 * 1) Displacement flush the E$ line corresponding to %addr.
407 * The first ldxa guarantees that the %addr is no longer in
408 * M, O, or E (goes to I or S (if instruction fetch also happens).
409 * 2) "Write" the data using a CAS %addr,%g0,%g0.
410 * The casxa guarantees a transition from I to M or S to M.
411 * 3) Displacement flush the E$ line corresponding to %addr.
412 * The second ldxa pushes the M line out of the ecache, into the
413 * writeback buffers, on the way to memory.
414 * 4) The "membar #Sync" pushes the cache line out of the writeback
415 * buffers onto the bus, on the way to dram finally.
416 *
417 * This is a modified version of the algorithm suggested by Gary Lauterbach.
418 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
419 * as modified, but then we found out that for spitfire, if it misses in the
420 * E$ it will probably install as an M, but if it hits in the E$, then it
421 * will stay E, if the store doesn't happen. So the first displacement flush
422 * should ensure that the CAS will miss in the E$. Arrgh.
423 */
424 ENTRY(scrubphys)
425 rdpr %pstate, %o4
426 andn %o4, PSTATE_IE | PSTATE_AM, %o5
427 wrpr %o5, %g0, %pstate ! clear IE, AM bits
428
429 GET_CPU_IMPL(%o5) ! Panther Ecache is flushed differently
430 cmp %o5, PANTHER_IMPL
431 bne scrubphys_1
432 nop
433 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5)
434 casxa [%o0]ASI_MEM, %g0, %g0
435 PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0)
436 b scrubphys_2
437 nop
438 scrubphys_1:
439 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
440 casxa [%o0]ASI_MEM, %g0, %g0
441 ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
442 scrubphys_2:
443 wrpr %g0, %o4, %pstate ! restore earlier pstate register value
444
445 retl
446 membar #Sync ! move the data out of the load buffer
447 SET_SIZE(scrubphys)
448
449
450 /*
451 * clearphys - Pass in the physical memory address of the checkblock
452 * that you want to push out, cleared with a recognizable pattern,
453 * from the ecache.
454 *
455 * To ensure that the ecc gets recalculated after the bad data is cleared,
456 * we must write out enough data to fill the w$ line (64 bytes). So we read
457 * in an entire ecache subblock's worth of data, and write it back out.
458 * Then we overwrite the 16 bytes of bad data with the pattern.
459 */
460 ENTRY(clearphys)
461 /* turn off IE, AM bits */
462 rdpr %pstate, %o4
463 andn %o4, PSTATE_IE | PSTATE_AM, %o5
464 wrpr %o5, %g0, %pstate
465
466 /* turn off NCEEN */
467 ldxa [%g0]ASI_ESTATE_ERR, %o5
468 andn %o5, EN_REG_NCEEN, %o3
469 stxa %o3, [%g0]ASI_ESTATE_ERR
470 membar #Sync
471
472 /* align address passed with 64 bytes subblock size */
473 mov CH_ECACHE_SUBBLK_SIZE, %o2
474 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
475
476 /* move the good data into the W$ */
477 clearphys_1:
478 subcc %o2, 8, %o2
479 ldxa [%g1 + %o2]ASI_MEM, %g2
497 nop
498 clearphys_2:
499 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
500 casxa [%o0]ASI_MEM, %g0, %g0
501 ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
502 clearphys_3:
503 /* clear the AFSR */
504 ldxa [%g0]ASI_AFSR, %o1
505 stxa %o1, [%g0]ASI_AFSR
506 membar #Sync
507
508 /* turn NCEEN back on */
509 stxa %o5, [%g0]ASI_ESTATE_ERR
510 membar #Sync
511
512 /* return and re-enable IE and AM */
513 retl
514 wrpr %g0, %o4, %pstate
515 SET_SIZE(clearphys)
516
517
518 /*
519 * Cheetah+ Ecache displacement flush the specified line from the E$
520 *
521 * For Panther, this means flushing the specified line from both the
522 * L2 cache and L3 cache.
523 *
524 * Register usage:
525 * %o0 - 64 bit physical address for flushing
526 * %o1 - Ecache set size
527 */
528 ENTRY(ecache_flush_line)
529
530 GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently
531 cmp %o3, PANTHER_IMPL
532 bne ecache_flush_line_1
533 nop
534
535 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4)
536 b ecache_flush_line_2
537 nop
538 ecache_flush_line_1:
539 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
540 ecache_flush_line_2:
541 retl
542 nop
543 SET_SIZE(ecache_flush_line)
544
545 ENTRY(set_afsr_ext)
546 set ASI_AFSR_EXT_VA, %o1
547 stxa %o0, [%o1]ASI_AFSR ! afsr_ext reg
548 membar #Sync
549 retl
550 nop
551 SET_SIZE(set_afsr_ext)
552
553
554 /*
555 * The CPU jumps here from the MMU exception handler if an ITLB parity
556 * error is detected and we are running on Panther.
557 *
558 * In this routine we collect diagnostic information and write it to our
559 * logout structure (if possible) and clear all ITLB entries that may have
560 * caused our parity trap.
561 * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
562 * and log any error messages. As for parameters to cpu_tlb_parity_error, we
563 * send two:
564 *
565 * %g2 - Contains the VA whose lookup in the ITLB caused the parity error
566 * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct,
567 * regardless of whether or not we actually used the logout struct.
568 *
569 * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
570 * parameters to the data contained in the logout structure in order to
571 * determine whether the logout information is valid for this particular
572 * error or not.
573 */
574 ENTRY_NP(itlb_parity_trap)
575 /*
576 * Collect important information about the trap which will be
577 * used as a parameter to the TL0 handler.
578 */
579 wr %g0, ASI_IMMU, %asi
580 rdpr %tpc, %g2 ! VA that caused the IMMU trap
581 ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page size
582 set PN_ITLB_PGSZ_MASK, %g4
583 and %g3, %g4, %g3
584 ldxa [MMU_TAG_ACCESS]%asi, %g4
585 set TAGREAD_CTX_MASK, %g5
586 and %g4, %g5, %g4
587 or %g4, %g3, %g3 ! 'or' in the trap context and
588 mov 1, %g4 ! add the IMMU flag to complete
589 sllx %g4, PN_TLO_INFO_IMMU_SHIFT, %g4
590 or %g4, %g3, %g3 ! the tlo_info field for logout
591 stxa %g0,[MMU_SFSR]%asi ! clear the SFSR
592 membar #Sync
593
676 membar #Sync
677
678 sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a
679 flush %g6 ! flush after writing MMU regs
680
681 /*
682 * at this point:
683 * %g2 - contains the VA whose lookup caused the trap
684 * %g3 - contains the tlo_info field
685 *
686 * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
687 * already at PIL 15. */
688 set cpu_tlb_parity_error, %g1
689 rdpr %pil, %g4
690 cmp %g4, PIL_14
691 movl %icc, PIL_14, %g4
692 ba sys_trap
693 nop
694 SET_SIZE(itlb_parity_trap)
695
696 /*
697 * The CPU jumps here from the MMU exception handler if a DTLB parity
698 * error is detected and we are running on Panther.
699 *
700 * In this routine we collect diagnostic information and write it to our
701 * logout structure (if possible) and clear all DTLB entries that may have
702 * caused our parity trap.
703 * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
704 * and log any error messages. As for parameters to cpu_tlb_parity_error, we
705 * send two:
706 *
707 * %g2 - Contains the VA whose lookup in the DTLB caused the parity error
708 * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct,
709 * regardless of whether or not we actually used the logout struct.
710 *
711 * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
712 * parameters to the data contained in the logout structure in order to
713 * determine whether the logout information is valid for this particular
714 * error or not.
715 */
716 ENTRY_NP(dtlb_parity_trap)
717 /*
718 * Collect important information about the trap which will be
719 * used as a parameter to the TL0 handler.
720 */
721 wr %g0, ASI_DMMU, %asi
722 ldxa [MMU_SFAR]%asi, %g2 ! VA that caused the IMMU trap
723 ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page sizes
724 set PN_DTLB_PGSZ_MASK, %g4
725 and %g3, %g4, %g3
726 ldxa [MMU_TAG_ACCESS]%asi, %g4
727 set TAGREAD_CTX_MASK, %g5 ! 'or' in the trap context
728 and %g4, %g5, %g4 ! to complete the tlo_info
729 or %g4, %g3, %g3 ! field for logout
730 stxa %g0,[MMU_SFSR]%asi ! clear the SFSR
731 membar #Sync
732
733 /*
734 * at this point:
735 * %g2 - contains the VA whose lookup caused the trap
880 sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a
881 flush %g6 ! flush after writing MMU regs
882
883 /*
884 * at this point:
885 * %g2 - contains the VA whose lookup caused the trap
886 * %g3 - contains the tlo_info field
887 *
888 * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
889 * already at PIL 15. We do this even for TL>1 traps since
890 * those will lead to a system panic.
891 */
892 set cpu_tlb_parity_error, %g1
893 rdpr %pil, %g4
894 cmp %g4, PIL_14
895 movl %icc, PIL_14, %g4
896 ba sys_trap
897 nop
898 SET_SIZE(dtlb_parity_trap)
899
900
901 /*
902 * Calculates the Panther TLB index based on a virtual address and page size
903 *
904 * Register usage:
905 * %o0 - virtual address whose index we want
906 * %o1 - Page Size of the TLB in question as encoded in the
907 * ASI_[D|I]MMU_TAG_ACCESS_EXT register.
908 */
909 ENTRY(pn_get_tlb_index)
910
911 PN_GET_TLB_INDEX(%o0, %o1)
912
913 retl
914 nop
915 SET_SIZE(pn_get_tlb_index)
916
917
918 /*
919 * For Panther CPUs we need to flush the IPB after any I$ or D$
920 * parity errors are detected.
921 */
922 ENTRY(flush_ipb)
923 clr %o0
924
925 flush_ipb_1:
926 stxa %g0, [%o0]ASI_IPB_TAG
927 membar #Sync
928 cmp %o0, PN_IPB_TAG_ADDR_MAX
929 blt flush_ipb_1
930 add %o0, PN_IPB_TAG_ADDR_LINESIZE, %o0
931
932 sethi %hi(FLUSH_ADDR), %o0
933 flush %o0
934 retl
935 nop
936 SET_SIZE(flush_ipb)
937
938
|