Print this page
restore sparc comments
de-linting of .s files


   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  *
  25  * Assembly code support for the Cheetah+ module
  26  */
  27 
  28 #pragma ident   "%Z%%M% %I%     %E% SMI"
  29 
  30 #if !defined(lint)
  31 #include "assym.h"
  32 #endif  /* lint */
  33 
  34 #include <sys/asm_linkage.h>
  35 #include <sys/mmu.h>
  36 #include <vm/hat_sfmmu.h>
  37 #include <sys/machparam.h>
  38 #include <sys/machcpuvar.h>
  39 #include <sys/machthread.h>
  40 #include <sys/machtrap.h>
  41 #include <sys/privregs.h>
  42 #include <sys/asm_linkage.h>
  43 #include <sys/trap.h>
  44 #include <sys/cheetahregs.h>
  45 #include <sys/us3_module.h>
  46 #include <sys/xc_impl.h>
  47 #include <sys/intreg.h>
  48 #include <sys/async.h>
  49 #include <sys/clock.h>
  50 #include <sys/cheetahasm.h>
  51 #include <sys/cmpregs.h>
  52 
  53 #ifdef TRAPTRACE
  54 #include <sys/traptrace.h>
  55 #endif /* TRAPTRACE */
  56 
  57 
  58 #if !defined(lint)
  59 
  60 /* BEGIN CSTYLED */
  61 
  62 /*
  63  * Cheetah+ version to reflush an Ecache line by index.
  64  *
  65  * By default we assume the Ecache is 2-way so we flush both
  66  * ways. Even if the cache is direct-mapped no harm will come
  67  * from performing the flush twice, apart from perhaps a performance
  68  * penalty.
  69  *
  70  * XXX - scr2 not used.
  71  */
  72 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2)                   \
  73         ldxa    [index]ASI_EC_DIAG, %g0;                                \
  74         ldxa    [index + ec_set_size]ASI_EC_DIAG, %g0;
  75 
  76 /*
  77  * Cheetah+ version of ecache_flush_line.  Uses Cheetah+ Ecache Displacement
  78  * Flush feature.
  79  */


 130  *              Output: Index into the L2$ of the line to be flushed.
 131  * l3_idx_out   Input: scratch register.
 132  *              Output: Index into the L3$ of the line to be flushed.
 133  * scr3         scratch register.
 134  * scr4         scratch register.
 135  *
 136  */
 137 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4)      \
 138         set     PN_L3_SET_SIZE, l2_idx_out;                                     \
 139         sub     l2_idx_out, 1, l2_idx_out;                                      \
 140         and     physaddr, l2_idx_out, l3_idx_out;                               \
 141         set     PN_L3_IDX_DISP_FLUSH, l2_idx_out;                               \
 142         or      l2_idx_out, l3_idx_out, l3_idx_out;                             \
 143         set     PN_L2_SET_SIZE, l2_idx_out;                                     \
 144         sub     l2_idx_out, 1, l2_idx_out;                                      \
 145         and     physaddr, l2_idx_out, l2_idx_out;                               \
 146         set     PN_L2_IDX_DISP_FLUSH, scr3;                                     \
 147         or      l2_idx_out, scr3, l2_idx_out;                                   \
 148         PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
 149 
 150 #endif  /* !lint */
 151 
 152 /*
 153  * Fast ECC error at TL>0 handler
 154  * We get here via trap 70 at TL>0->Software trap 0 at TL>0.  We enter
 155  * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
 156  * For a complete description of the Fast ECC at TL>0 handling see the
 157  * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
 158  * us3_common_asm.s
 159  */
 160 #if defined(lint)
 161 
 162 void
 163 fast_ecc_tl1_err(void)
 164 {}
 165 
 166 #else   /* lint */
 167 
 168         .section ".text"
 169         .align  64
 170         ENTRY_NP(fast_ecc_tl1_err)
 171 
 172         /*
 173          * This macro turns off the D$/I$ if they are on and saves their
 174          * original state in ch_err_tl1_tmp, saves all the %g registers in the
 175          * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
 176          * the %tpc in ch_err_tl1_tpc.  At the end of this macro, %g1 will
 177          * point to the ch_err_tl1_data structure and the original D$/I$ state
 178          * will be saved in ch_err_tl1_tmp.  All %g registers except for %g1
 179          * will be available.
 180          */
 181         CH_ERR_TL1_FECC_ENTER;
 182 
 183         /*
 184          * Get the diagnostic logout data.  %g4 must be initialized to
 185          * current CEEN state, %g5 must point to logout structure in
 186          * ch_err_tl1_data_t.  %g3 will contain the nesting count upon
 187          * return.


 396         or      %g3, %g4, %g3           ! %g3 = original + current AFSR_EXT
 397         btst    C_AFSR_L3_WDU, %g3      ! L3_WDU in original or current AFSR?
 398         bnz     %xcc, fecc_tl1_err      ! panic (saw L3_WDU and UCU or L3_UCU)
 399           nop
 400 6:
 401         /*
 402          * We fall into this macro if we've successfully logged the error in
 403          * the ch_err_tl1_data structure and want the PIL15 softint to pick
 404          * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
 405          * Restores the %g registers and issues retry.
 406          */
 407         CH_ERR_TL1_EXIT;
 408 
 409         /*
 410          * Establish panic exit label.
 411          */
 412         CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
 413 
 414         SET_SIZE(fast_ecc_tl1_err)
 415 
 416 #endif  /* lint */
 417 
 418 
 419 #if defined(lint)
 420 /*
 421  * scrubphys - Pass in the aligned physical memory address
 422  * that you want to scrub, along with the ecache set size.
 423  *
 424  *      1) Displacement flush the E$ line corresponding to %addr.
 425  *         The first ldxa guarantees that the %addr is no longer in
 426  *         M, O, or E (goes to I or S (if instruction fetch also happens).
 427  *      2) "Write" the data using a CAS %addr,%g0,%g0.
 428  *         The casxa guarantees a transition from I to M or S to M.
 429  *      3) Displacement flush the E$ line corresponding to %addr.
 430  *         The second ldxa pushes the M line out of the ecache, into the
 431  *         writeback buffers, on the way to memory.
 432  *      4) The "membar #Sync" pushes the cache line out of the writeback
 433  *         buffers onto the bus, on the way to dram finally.
 434  *
 435  * This is a modified version of the algorithm suggested by Gary Lauterbach.
 436  * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
 437  * as modified, but then we found out that for spitfire, if it misses in the
 438  * E$ it will probably install as an M, but if it hits in the E$, then it
 439  * will stay E, if the store doesn't happen. So the first displacement flush
 440  * should ensure that the CAS will miss in the E$.  Arrgh.
 441  */
 442 /* ARGSUSED */
 443 void
 444 scrubphys(uint64_t paddr, int ecache_set_size)
 445 {}
 446 
 447 #else   /* lint */
 448         ENTRY(scrubphys)
 449         rdpr    %pstate, %o4
 450         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 451         wrpr    %o5, %g0, %pstate       ! clear IE, AM bits
 452 
 453         GET_CPU_IMPL(%o5)               ! Panther Ecache is flushed differently
 454         cmp     %o5, PANTHER_IMPL
 455         bne     scrubphys_1
 456           nop
 457         PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5)
 458         casxa   [%o0]ASI_MEM, %g0, %g0
 459         PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0)
 460         b       scrubphys_2
 461           nop
 462 scrubphys_1:
 463         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 464         casxa   [%o0]ASI_MEM, %g0, %g0
 465         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 466 scrubphys_2:
 467         wrpr    %g0, %o4, %pstate       ! restore earlier pstate register value
 468 
 469         retl
 470         membar  #Sync                   ! move the data out of the load buffer
 471         SET_SIZE(scrubphys)
 472 
 473 #endif  /* lint */
 474 
 475 
 476 #if defined(lint)
 477 /*
 478  * clearphys - Pass in the physical memory address of the checkblock
 479  * that you want to push out, cleared with a recognizable pattern,
 480  * from the ecache.
 481  *
 482  * To ensure that the ecc gets recalculated after the bad data is cleared,
 483  * we must write out enough data to fill the w$ line (64 bytes). So we read
 484  * in an entire ecache subblock's worth of data, and write it back out.
 485  * Then we overwrite the 16 bytes of bad data with the pattern.
 486  */
 487 /* ARGSUSED */
 488 void
 489 clearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize)
 490 {
 491 }
 492 
 493 #else   /* lint */
 494         ENTRY(clearphys)
 495         /* turn off IE, AM bits */
 496         rdpr    %pstate, %o4
 497         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 498         wrpr    %o5, %g0, %pstate
 499 
 500         /* turn off NCEEN */
 501         ldxa    [%g0]ASI_ESTATE_ERR, %o5
 502         andn    %o5, EN_REG_NCEEN, %o3
 503         stxa    %o3, [%g0]ASI_ESTATE_ERR
 504         membar  #Sync
 505 
 506         /* align address passed with 64 bytes subblock size */
 507         mov     CH_ECACHE_SUBBLK_SIZE, %o2
 508         andn    %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
 509 
 510         /* move the good data into the W$ */    
 511 clearphys_1:
 512         subcc   %o2, 8, %o2
 513         ldxa    [%g1 + %o2]ASI_MEM, %g2


 531           nop
 532 clearphys_2:
 533         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 534         casxa   [%o0]ASI_MEM, %g0, %g0
 535         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 536 clearphys_3:
 537         /* clear the AFSR */
 538         ldxa    [%g0]ASI_AFSR, %o1
 539         stxa    %o1, [%g0]ASI_AFSR
 540         membar  #Sync
 541 
 542         /* turn NCEEN back on */
 543         stxa    %o5, [%g0]ASI_ESTATE_ERR
 544         membar  #Sync
 545 
 546         /* return and re-enable IE and AM */
 547         retl
 548           wrpr  %g0, %o4, %pstate
 549         SET_SIZE(clearphys)
 550 
 551 #endif  /* lint */
 552 
 553 
 554 #if defined(lint)
 555 /*
 556  * Cheetah+ Ecache displacement flush the specified line from the E$
 557  *
 558  * For Panther, this means flushing the specified line from both the
 559  * L2 cache and L3 cache.
 560  *
 561  * Register usage:
 562  *      %o0 - 64 bit physical address for flushing
 563  *      %o1 - Ecache set size
 564  */
 565 /*ARGSUSED*/
 566 void
 567 ecache_flush_line(uint64_t flushaddr, int ec_set_size)
 568 {
 569 }
 570 #else   /* lint */
 571         ENTRY(ecache_flush_line)
 572 
 573         GET_CPU_IMPL(%o3)               ! Panther Ecache is flushed differently
 574         cmp     %o3, PANTHER_IMPL
 575         bne     ecache_flush_line_1
 576           nop
 577 
 578         PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4)
 579         b       ecache_flush_line_2
 580           nop
 581 ecache_flush_line_1:
 582         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 583 ecache_flush_line_2:
 584         retl
 585           nop
 586         SET_SIZE(ecache_flush_line)
 587 #endif  /* lint */
 588 
 589 #if defined(lint)
 590 void
 591 set_afsr_ext(uint64_t afsr_ext)
 592 {
 593         afsr_ext = afsr_ext;
 594 }
 595 #else /* lint */
 596 
 597         ENTRY(set_afsr_ext)
 598         set     ASI_AFSR_EXT_VA, %o1
 599         stxa    %o0, [%o1]ASI_AFSR              ! afsr_ext reg
 600         membar  #Sync
 601         retl
 602         nop
 603         SET_SIZE(set_afsr_ext)
 604 
 605 #endif /* lint */
 606 
 607 
 608 #if defined(lint)
 609 /*
 610  * The CPU jumps here from the MMU exception handler if an ITLB parity
 611  * error is detected and we are running on Panther.
 612  *
 613  * In this routine we collect diagnostic information and write it to our
 614  * logout structure (if possible) and clear all ITLB entries that may have
 615  * caused our parity trap.
 616  * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
 617  * and log any error messages. As for parameters to cpu_tlb_parity_error, we
 618  * send two:
 619  *
 620  * %g2  - Contains the VA whose lookup in the ITLB caused the parity error
 621  * %g3  - Contains the tlo_info field of the pn_tlb_logout logout struct,
 622  *        regardless of whether or not we actually used the logout struct.
 623  *
 624  * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
 625  * parameters to the data contained in the logout structure in order to
 626  * determine whether the logout information is valid for this particular
 627  * error or not.
 628  */
 629 void
 630 itlb_parity_trap(void)
 631 {}
 632 
 633 #else   /* lint */
 634 
 635         ENTRY_NP(itlb_parity_trap)
 636         /*
 637          * Collect important information about the trap which will be
 638          * used as a parameter to the TL0 handler.
 639          */
 640         wr      %g0, ASI_IMMU, %asi
 641         rdpr    %tpc, %g2                       ! VA that caused the IMMU trap
 642         ldxa    [MMU_TAG_ACCESS_EXT]%asi, %g3   ! read the trap VA page size
 643         set     PN_ITLB_PGSZ_MASK, %g4
 644         and     %g3, %g4, %g3
 645         ldxa    [MMU_TAG_ACCESS]%asi, %g4
 646         set     TAGREAD_CTX_MASK, %g5
 647         and     %g4, %g5, %g4
 648         or      %g4, %g3, %g3                   ! 'or' in the trap context and
 649         mov     1, %g4                          ! add the IMMU flag to complete
 650         sllx    %g4, PN_TLO_INFO_IMMU_SHIFT, %g4
 651         or      %g4, %g3, %g3                   ! the tlo_info field for logout
 652         stxa    %g0,[MMU_SFSR]%asi              ! clear the SFSR
 653         membar  #Sync
 654 


 737         membar  #Sync
 738 
 739         sethi   %hi(FLUSH_ADDR), %g6            ! PRM says we need to issue a
 740         flush   %g6                             ! flush after writing MMU regs
 741 
 742         /*
 743          * at this point:
 744          *    %g2 - contains the VA whose lookup caused the trap
 745          *    %g3 - contains the tlo_info field
 746          *
 747          * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
 748          * already at PIL 15.    */
 749         set     cpu_tlb_parity_error, %g1
 750         rdpr    %pil, %g4
 751         cmp     %g4, PIL_14
 752         movl    %icc, PIL_14, %g4
 753         ba      sys_trap
 754           nop
 755         SET_SIZE(itlb_parity_trap)
 756 
 757 #endif  /* lint */
 758 
 759 #if defined(lint)
 760 /*
 761  * The CPU jumps here from the MMU exception handler if a DTLB parity
 762  * error is detected and we are running on Panther.
 763  *
 764  * In this routine we collect diagnostic information and write it to our
 765  * logout structure (if possible) and clear all DTLB entries that may have
 766  * caused our parity trap.
 767  * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
 768  * and log any error messages. As for parameters to cpu_tlb_parity_error, we
 769  * send two:
 770  *
 771  * %g2  - Contains the VA whose lookup in the DTLB caused the parity error
 772  * %g3  - Contains the tlo_info field of the pn_tlb_logout logout struct,
 773  *        regardless of whether or not we actually used the logout struct.
 774  *
 775  * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
 776  * parameters to the data contained in the logout structure in order to
 777  * determine whether the logout information is valid for this particular
 778  * error or not.
 779  */
 780 void
 781 dtlb_parity_trap(void)
 782 {}
 783 
 784 #else   /* lint */
 785 
 786         ENTRY_NP(dtlb_parity_trap)
 787         /*
 788          * Collect important information about the trap which will be
 789          * used as a parameter to the TL0 handler.
 790          */
 791         wr      %g0, ASI_DMMU, %asi
 792         ldxa    [MMU_SFAR]%asi, %g2             ! VA that caused the IMMU trap
 793         ldxa    [MMU_TAG_ACCESS_EXT]%asi, %g3   ! read the trap VA page sizes
 794         set     PN_DTLB_PGSZ_MASK, %g4
 795         and     %g3, %g4, %g3
 796         ldxa    [MMU_TAG_ACCESS]%asi, %g4
 797         set     TAGREAD_CTX_MASK, %g5           ! 'or' in the trap context
 798         and     %g4, %g5, %g4                   ! to complete the tlo_info
 799         or      %g4, %g3, %g3                   ! field for logout
 800         stxa    %g0,[MMU_SFSR]%asi              ! clear the SFSR
 801         membar  #Sync
 802 
 803         /*
 804          * at this point:
 805          *    %g2 - contains the VA whose lookup caused the trap


 950         sethi   %hi(FLUSH_ADDR), %g6            ! PRM says we need to issue a
 951         flush   %g6                             ! flush after writing MMU regs
 952 
 953         /*
 954          * at this point:
 955          *    %g2 - contains the VA whose lookup caused the trap
 956          *    %g3 - contains the tlo_info field
 957          *
 958          * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
 959          * already at PIL 15. We do this even for TL>1 traps since
 960          * those will lead to a system panic.
 961          */
 962         set     cpu_tlb_parity_error, %g1
 963         rdpr    %pil, %g4
 964         cmp     %g4, PIL_14
 965         movl    %icc, PIL_14, %g4
 966         ba      sys_trap
 967           nop
 968         SET_SIZE(dtlb_parity_trap)
 969 
 970 #endif  /* lint */
 971 
 972 
 973 #if defined(lint)
 974 /*
 975  * Calculates the Panther TLB index based on a virtual address and page size
 976  *
 977  * Register usage:
 978  *      %o0 - virtual address whose index we want
 979  *      %o1 - Page Size of the TLB in question as encoded in the
 980  *            ASI_[D|I]MMU_TAG_ACCESS_EXT register.
 981  */
 982 uint64_t
 983 pn_get_tlb_index(uint64_t va, uint64_t pg_sz)
 984 {
 985         return ((va + pg_sz)-(va + pg_sz));
 986 }
 987 #else   /* lint */
 988         ENTRY(pn_get_tlb_index)
 989 
 990         PN_GET_TLB_INDEX(%o0, %o1)
 991 
 992         retl
 993           nop
 994         SET_SIZE(pn_get_tlb_index)
 995 #endif  /* lint */
 996 
 997 
 998 #if defined(lint)
 999 /*
1000  * For Panther CPUs we need to flush the IPB after any I$ or D$
1001  * parity errors are detected.
1002  */
1003 void
1004 flush_ipb(void)
1005 { return; }
1006 
1007 #else   /* lint */
1008 
1009         ENTRY(flush_ipb)
1010         clr     %o0
1011 
1012 flush_ipb_1:
1013         stxa    %g0, [%o0]ASI_IPB_TAG
1014         membar  #Sync
1015         cmp     %o0, PN_IPB_TAG_ADDR_MAX
1016         blt     flush_ipb_1
1017           add   %o0, PN_IPB_TAG_ADDR_LINESIZE,  %o0
1018 
1019         sethi   %hi(FLUSH_ADDR), %o0
1020         flush   %o0
1021         retl
1022         nop
1023         SET_SIZE(flush_ipb)
1024 
1025 #endif  /* lint */
1026 
1027 


   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  *
  25  * Assembly code support for the Cheetah+ module
  26  */
  27 



  28 #include "assym.h"

  29 
  30 #include <sys/asm_linkage.h>
  31 #include <sys/mmu.h>
  32 #include <vm/hat_sfmmu.h>
  33 #include <sys/machparam.h>
  34 #include <sys/machcpuvar.h>
  35 #include <sys/machthread.h>
  36 #include <sys/machtrap.h>
  37 #include <sys/privregs.h>
  38 #include <sys/asm_linkage.h>
  39 #include <sys/trap.h>
  40 #include <sys/cheetahregs.h>
  41 #include <sys/us3_module.h>
  42 #include <sys/xc_impl.h>
  43 #include <sys/intreg.h>
  44 #include <sys/async.h>
  45 #include <sys/clock.h>
  46 #include <sys/cheetahasm.h>
  47 #include <sys/cmpregs.h>
  48 
  49 #ifdef TRAPTRACE
  50 #include <sys/traptrace.h>
  51 #endif /* TRAPTRACE */
  52 
  53 


  54 /* BEGIN CSTYLED */
  55 
  56 /*
  57  * Cheetah+ version to reflush an Ecache line by index.
  58  *
  59  * By default we assume the Ecache is 2-way so we flush both
  60  * ways. Even if the cache is direct-mapped no harm will come
  61  * from performing the flush twice, apart from perhaps a performance
  62  * penalty.
  63  *
  64  * XXX - scr2 not used.
  65  */
  66 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2)                   \
  67         ldxa    [index]ASI_EC_DIAG, %g0;                                \
  68         ldxa    [index + ec_set_size]ASI_EC_DIAG, %g0;
  69 
  70 /*
  71  * Cheetah+ version of ecache_flush_line.  Uses Cheetah+ Ecache Displacement
  72  * Flush feature.
  73  */


 124  *              Output: Index into the L2$ of the line to be flushed.
 125  * l3_idx_out   Input: scratch register.
 126  *              Output: Index into the L3$ of the line to be flushed.
 127  * scr3         scratch register.
 128  * scr4         scratch register.
 129  *
 130  */
 131 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4)      \
 132         set     PN_L3_SET_SIZE, l2_idx_out;                                     \
 133         sub     l2_idx_out, 1, l2_idx_out;                                      \
 134         and     physaddr, l2_idx_out, l3_idx_out;                               \
 135         set     PN_L3_IDX_DISP_FLUSH, l2_idx_out;                               \
 136         or      l2_idx_out, l3_idx_out, l3_idx_out;                             \
 137         set     PN_L2_SET_SIZE, l2_idx_out;                                     \
 138         sub     l2_idx_out, 1, l2_idx_out;                                      \
 139         and     physaddr, l2_idx_out, l2_idx_out;                               \
 140         set     PN_L2_IDX_DISP_FLUSH, scr3;                                     \
 141         or      l2_idx_out, scr3, l2_idx_out;                                   \
 142         PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
 143 


 144 /*
 145  * Fast ECC error at TL>0 handler
 146  * We get here via trap 70 at TL>0->Software trap 0 at TL>0.  We enter
 147  * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
 148  * For a complete description of the Fast ECC at TL>0 handling see the
 149  * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
 150  * us3_common_asm.s
 151  */

 152 






 153         .section ".text"
 154         .align  64
 155         ENTRY_NP(fast_ecc_tl1_err)
 156 
 157         /*
 158          * This macro turns off the D$/I$ if they are on and saves their
 159          * original state in ch_err_tl1_tmp, saves all the %g registers in the
 160          * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
 161          * the %tpc in ch_err_tl1_tpc.  At the end of this macro, %g1 will
 162          * point to the ch_err_tl1_data structure and the original D$/I$ state
 163          * will be saved in ch_err_tl1_tmp.  All %g registers except for %g1
 164          * will be available.
 165          */
 166         CH_ERR_TL1_FECC_ENTER;
 167 
 168         /*
 169          * Get the diagnostic logout data.  %g4 must be initialized to
 170          * current CEEN state, %g5 must point to logout structure in
 171          * ch_err_tl1_data_t.  %g3 will contain the nesting count upon
 172          * return.


 381         or      %g3, %g4, %g3           ! %g3 = original + current AFSR_EXT
 382         btst    C_AFSR_L3_WDU, %g3      ! L3_WDU in original or current AFSR?
 383         bnz     %xcc, fecc_tl1_err      ! panic (saw L3_WDU and UCU or L3_UCU)
 384           nop
 385 6:
 386         /*
 387          * We fall into this macro if we've successfully logged the error in
 388          * the ch_err_tl1_data structure and want the PIL15 softint to pick
 389          * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
 390          * Restores the %g registers and issues retry.
 391          */
 392         CH_ERR_TL1_EXIT;
 393 
 394         /*
 395          * Establish panic exit label.
 396          */
 397         CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
 398 
 399         SET_SIZE(fast_ecc_tl1_err)
 400 

 401 


 402 /*
 403  * scrubphys - Pass in the aligned physical memory address
 404  * that you want to scrub, along with the ecache set size.
 405  *
 406  *      1) Displacement flush the E$ line corresponding to %addr.
 407  *         The first ldxa guarantees that the %addr is no longer in
 408  *         M, O, or E (goes to I or S (if instruction fetch also happens).
 409  *      2) "Write" the data using a CAS %addr,%g0,%g0.
 410  *         The casxa guarantees a transition from I to M or S to M.
 411  *      3) Displacement flush the E$ line corresponding to %addr.
 412  *         The second ldxa pushes the M line out of the ecache, into the
 413  *         writeback buffers, on the way to memory.
 414  *      4) The "membar #Sync" pushes the cache line out of the writeback
 415  *         buffers onto the bus, on the way to dram finally.
 416  *
 417  * This is a modified version of the algorithm suggested by Gary Lauterbach.
 418  * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
 419  * as modified, but then we found out that for spitfire, if it misses in the
 420  * E$ it will probably install as an M, but if it hits in the E$, then it
 421  * will stay E, if the store doesn't happen. So the first displacement flush
 422  * should ensure that the CAS will miss in the E$.  Arrgh.
 423  */






 424         ENTRY(scrubphys)
 425         rdpr    %pstate, %o4
 426         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 427         wrpr    %o5, %g0, %pstate       ! clear IE, AM bits
 428 
 429         GET_CPU_IMPL(%o5)               ! Panther Ecache is flushed differently
 430         cmp     %o5, PANTHER_IMPL
 431         bne     scrubphys_1
 432           nop
 433         PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5)
 434         casxa   [%o0]ASI_MEM, %g0, %g0
 435         PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0)
 436         b       scrubphys_2
 437           nop
 438 scrubphys_1:
 439         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 440         casxa   [%o0]ASI_MEM, %g0, %g0
 441         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 442 scrubphys_2:
 443         wrpr    %g0, %o4, %pstate       ! restore earlier pstate register value
 444 
 445         retl
 446         membar  #Sync                   ! move the data out of the load buffer
 447         SET_SIZE(scrubphys)
 448 

 449 


 450 /*
 451  * clearphys - Pass in the physical memory address of the checkblock
 452  * that you want to push out, cleared with a recognizable pattern,
 453  * from the ecache.
 454  *
 455  * To ensure that the ecc gets recalculated after the bad data is cleared,
 456  * we must write out enough data to fill the w$ line (64 bytes). So we read
 457  * in an entire ecache subblock's worth of data, and write it back out.
 458  * Then we overwrite the 16 bytes of bad data with the pattern.
 459  */







 460         ENTRY(clearphys)
 461         /* turn off IE, AM bits */
 462         rdpr    %pstate, %o4
 463         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 464         wrpr    %o5, %g0, %pstate
 465 
 466         /* turn off NCEEN */
 467         ldxa    [%g0]ASI_ESTATE_ERR, %o5
 468         andn    %o5, EN_REG_NCEEN, %o3
 469         stxa    %o3, [%g0]ASI_ESTATE_ERR
 470         membar  #Sync
 471 
 472         /* align address passed with 64 bytes subblock size */
 473         mov     CH_ECACHE_SUBBLK_SIZE, %o2
 474         andn    %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
 475 
 476         /* move the good data into the W$ */    
 477 clearphys_1:
 478         subcc   %o2, 8, %o2
 479         ldxa    [%g1 + %o2]ASI_MEM, %g2


 497           nop
 498 clearphys_2:
 499         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 500         casxa   [%o0]ASI_MEM, %g0, %g0
 501         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 502 clearphys_3:
 503         /* clear the AFSR */
 504         ldxa    [%g0]ASI_AFSR, %o1
 505         stxa    %o1, [%g0]ASI_AFSR
 506         membar  #Sync
 507 
 508         /* turn NCEEN back on */
 509         stxa    %o5, [%g0]ASI_ESTATE_ERR
 510         membar  #Sync
 511 
 512         /* return and re-enable IE and AM */
 513         retl
 514           wrpr  %g0, %o4, %pstate
 515         SET_SIZE(clearphys)
 516 

 517 


 518 /*
 519  * Cheetah+ Ecache displacement flush the specified line from the E$
 520  *
 521  * For Panther, this means flushing the specified line from both the
 522  * L2 cache and L3 cache.
 523  *
 524  * Register usage:
 525  *      %o0 - 64 bit physical address for flushing
 526  *      %o1 - Ecache set size
 527  */






 528         ENTRY(ecache_flush_line)
 529 
 530         GET_CPU_IMPL(%o3)               ! Panther Ecache is flushed differently
 531         cmp     %o3, PANTHER_IMPL
 532         bne     ecache_flush_line_1
 533           nop
 534 
 535         PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4)
 536         b       ecache_flush_line_2
 537           nop
 538 ecache_flush_line_1:
 539         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 540 ecache_flush_line_2:
 541         retl
 542           nop
 543         SET_SIZE(ecache_flush_line)

 544 








 545         ENTRY(set_afsr_ext)
 546         set     ASI_AFSR_EXT_VA, %o1
 547         stxa    %o0, [%o1]ASI_AFSR              ! afsr_ext reg
 548         membar  #Sync
 549         retl
 550         nop
 551         SET_SIZE(set_afsr_ext)
 552 

 553 


 554 /*
 555  * The CPU jumps here from the MMU exception handler if an ITLB parity
 556  * error is detected and we are running on Panther.
 557  *
 558  * In this routine we collect diagnostic information and write it to our
 559  * logout structure (if possible) and clear all ITLB entries that may have
 560  * caused our parity trap.
 561  * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
 562  * and log any error messages. As for parameters to cpu_tlb_parity_error, we
 563  * send two:
 564  *
 565  * %g2  - Contains the VA whose lookup in the ITLB caused the parity error
 566  * %g3  - Contains the tlo_info field of the pn_tlb_logout logout struct,
 567  *        regardless of whether or not we actually used the logout struct.
 568  *
 569  * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
 570  * parameters to the data contained in the logout structure in order to
 571  * determine whether the logout information is valid for this particular
 572  * error or not.
 573  */






 574         ENTRY_NP(itlb_parity_trap)
 575         /*
 576          * Collect important information about the trap which will be
 577          * used as a parameter to the TL0 handler.
 578          */
 579         wr      %g0, ASI_IMMU, %asi
 580         rdpr    %tpc, %g2                       ! VA that caused the IMMU trap
 581         ldxa    [MMU_TAG_ACCESS_EXT]%asi, %g3   ! read the trap VA page size
 582         set     PN_ITLB_PGSZ_MASK, %g4
 583         and     %g3, %g4, %g3
 584         ldxa    [MMU_TAG_ACCESS]%asi, %g4
 585         set     TAGREAD_CTX_MASK, %g5
 586         and     %g4, %g5, %g4
 587         or      %g4, %g3, %g3                   ! 'or' in the trap context and
 588         mov     1, %g4                          ! add the IMMU flag to complete
 589         sllx    %g4, PN_TLO_INFO_IMMU_SHIFT, %g4
 590         or      %g4, %g3, %g3                   ! the tlo_info field for logout
 591         stxa    %g0,[MMU_SFSR]%asi              ! clear the SFSR
 592         membar  #Sync
 593 


 676         membar  #Sync
 677 
 678         sethi   %hi(FLUSH_ADDR), %g6            ! PRM says we need to issue a
 679         flush   %g6                             ! flush after writing MMU regs
 680 
 681         /*
 682          * at this point:
 683          *    %g2 - contains the VA whose lookup caused the trap
 684          *    %g3 - contains the tlo_info field
 685          *
 686          * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
 687          * already at PIL 15.    */
 688         set     cpu_tlb_parity_error, %g1
 689         rdpr    %pil, %g4
 690         cmp     %g4, PIL_14
 691         movl    %icc, PIL_14, %g4
 692         ba      sys_trap
 693           nop
 694         SET_SIZE(itlb_parity_trap)
 695 



 696 /*
 697  * The CPU jumps here from the MMU exception handler if a DTLB parity
 698  * error is detected and we are running on Panther.
 699  *
 700  * In this routine we collect diagnostic information and write it to our
 701  * logout structure (if possible) and clear all DTLB entries that may have
 702  * caused our parity trap.
 703  * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
 704  * and log any error messages. As for parameters to cpu_tlb_parity_error, we
 705  * send two:
 706  *
 707  * %g2  - Contains the VA whose lookup in the DTLB caused the parity error
 708  * %g3  - Contains the tlo_info field of the pn_tlb_logout logout struct,
 709  *        regardless of whether or not we actually used the logout struct.
 710  *
 711  * In the TL0 handler (cpu_tlb_parity_error) we will compare those two
 712  * parameters to the data contained in the logout structure in order to
 713  * determine whether the logout information is valid for this particular
 714  * error or not.
 715  */






 716         ENTRY_NP(dtlb_parity_trap)
 717         /*
 718          * Collect important information about the trap which will be
 719          * used as a parameter to the TL0 handler.
 720          */
 721         wr      %g0, ASI_DMMU, %asi
 722         ldxa    [MMU_SFAR]%asi, %g2             ! VA that caused the IMMU trap
 723         ldxa    [MMU_TAG_ACCESS_EXT]%asi, %g3   ! read the trap VA page sizes
 724         set     PN_DTLB_PGSZ_MASK, %g4
 725         and     %g3, %g4, %g3
 726         ldxa    [MMU_TAG_ACCESS]%asi, %g4
 727         set     TAGREAD_CTX_MASK, %g5           ! 'or' in the trap context
 728         and     %g4, %g5, %g4                   ! to complete the tlo_info
 729         or      %g4, %g3, %g3                   ! field for logout
 730         stxa    %g0,[MMU_SFSR]%asi              ! clear the SFSR
 731         membar  #Sync
 732 
 733         /*
 734          * at this point:
 735          *    %g2 - contains the VA whose lookup caused the trap


 880         sethi   %hi(FLUSH_ADDR), %g6            ! PRM says we need to issue a
 881         flush   %g6                             ! flush after writing MMU regs
 882 
 883         /*
 884          * at this point:
 885          *    %g2 - contains the VA whose lookup caused the trap
 886          *    %g3 - contains the tlo_info field
 887          *
 888          * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
 889          * already at PIL 15. We do this even for TL>1 traps since
 890          * those will lead to a system panic.
 891          */
 892         set     cpu_tlb_parity_error, %g1
 893         rdpr    %pil, %g4
 894         cmp     %g4, PIL_14
 895         movl    %icc, PIL_14, %g4
 896         ba      sys_trap
 897           nop
 898         SET_SIZE(dtlb_parity_trap)
 899 

 900 


 901 /*
 902  * Calculates the Panther TLB index based on a virtual address and page size
 903  *
 904  * Register usage:
 905  *      %o0 - virtual address whose index we want
 906  *      %o1 - Page Size of the TLB in question as encoded in the
 907  *            ASI_[D|I]MMU_TAG_ACCESS_EXT register.
 908  */






 909         ENTRY(pn_get_tlb_index)
 910 
 911         PN_GET_TLB_INDEX(%o0, %o1)
 912 
 913         retl
 914           nop
 915         SET_SIZE(pn_get_tlb_index)

 916 
 917 

 918 /*
 919  * For Panther CPUs we need to flush the IPB after any I$ or D$
 920  * parity errors are detected.
 921  */






 922         ENTRY(flush_ipb)
 923         clr     %o0
 924 
 925 flush_ipb_1:
 926         stxa    %g0, [%o0]ASI_IPB_TAG
 927         membar  #Sync
 928         cmp     %o0, PN_IPB_TAG_ADDR_MAX
 929         blt     flush_ipb_1
 930           add   %o0, PN_IPB_TAG_ADDR_LINESIZE,  %o0
 931 
 932         sethi   %hi(FLUSH_ADDR), %o0
 933         flush   %o0
 934         retl
 935         nop
 936         SET_SIZE(flush_ipb)
 937 

 938