1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Assembly code support for the Cheetah module
  27  */
  28 
  29 #include "assym.h"
  30 
  31 #include <sys/asm_linkage.h>
  32 #include <sys/mmu.h>
  33 #include <vm/hat_sfmmu.h>
  34 #include <sys/machparam.h>
  35 #include <sys/machcpuvar.h>
  36 #include <sys/machthread.h>
  37 #include <sys/machtrap.h>
  38 #include <sys/privregs.h>
  39 #include <sys/asm_linkage.h>
  40 #include <sys/trap.h>
  41 #include <sys/cheetahregs.h>
  42 #include <sys/us3_module.h>
  43 #include <sys/xc_impl.h>
  44 #include <sys/intreg.h>
  45 #include <sys/async.h>
  46 #include <sys/clock.h>
  47 #include <sys/cheetahasm.h>
  48 
  49 #ifdef TRAPTRACE
  50 #include <sys/traptrace.h>
  51 #endif /* TRAPTRACE */
  52 
  53 /* BEGIN CSTYLED */
  54 
  55 /*
  56  * Cheetah version to flush an Ecache line by index (aliased address)
  57  */
  58 #define ECACHE_REFLUSH_LINE(ecache_size, alias_address, scr2)           \
  59         ldxa    [alias_address]ASI_MEM, %g0
  60 
  61 #define ECACHE_FLUSH_LINE(physaddr, ecache_size, scr1, scr2)            \
  62         xor     physaddr, ecache_size, scr1;                            \
  63         add     ecache_size, ecache_size, scr2;                         \
  64         sub     scr2, 1, scr2;                                          \
  65         and     scr1, scr2, scr1;                                       \
  66         ASM_LDX(scr2, ecache_flushaddr);                                \
  67         add     scr1, scr2, scr1;                                       \
  68         ECACHE_REFLUSH_LINE(ecache_size, scr1, scr2)
  69 
  70 /* END CSTYLED */
  71 
  72 
  73 /*
  74  * Fast ECC error at TL>0 handler
  75  * We get here via trap 70 at TL>0->Software trap 0 at TL>0.  We enter
  76  * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
  77  * For a complete description of the Fast ECC at TL>0 handling see the
  78  * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
  79  * us3_common_asm.s
  80  */
  81 
  82         .section ".text"
  83         .align  64
  84         ENTRY_NP(fast_ecc_tl1_err)
  85 
  86         /*
  87          * This macro turns off the D$/I$ if they are on and saves their
  88          * original state in ch_err_tl1_tmp, saves all the %g registers in the
  89          * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
  90          * the %tpc in ch_err_tl1_tpc.  At the end of this macro, %g1 will
  91          * point to the ch_err_tl1_data structure and the original D$/I$ state
  92          * will be saved in ch_err_tl1_tmp.  All %g registers except for %g1
  93          * will be available.
  94          */
  95         CH_ERR_TL1_FECC_ENTER;
  96 
  97         /*
  98          * Get the diagnostic logout data.  %g4 must be initialized to
  99          * current CEEN state, %g5 must point to logout structure in
 100          * ch_err_tl1_data_t.  %g3 will contain the nesting count upon
 101          * return.
 102          */
 103         ldxa    [%g0]ASI_ESTATE_ERR, %g4
 104         and     %g4, EN_REG_CEEN, %g4
 105         add     %g1, CH_ERR_TL1_LOGOUT, %g5
 106         DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4)
 107 
 108         /*
 109          * If the logout nesting count is exceeded, we're probably
 110          * not making any progress, try to panic instead.
 111          */
 112         cmp     %g3, CLO_NESTING_MAX
 113         bge     fecc_tl1_err
 114           nop
 115 
 116         /*
 117          * Save the current CEEN and NCEEN state in %g7 and turn them off
 118          * before flushing the Ecache.
 119          */
 120         ldxa    [%g0]ASI_ESTATE_ERR, %g7
 121         andn    %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5
 122         stxa    %g5, [%g0]ASI_ESTATE_ERR
 123         membar  #Sync
 124 
 125         /*
 126          * Flush the Ecache, using the largest possible cache size with the
 127          * smallest possible line size since we can't get the actual sizes
 128          * from the cpu_node due to DTLB misses.
 129          */
 130         set     CH_ECACHE_8M_SIZE, %g4
 131         set     CH_ECACHE_MIN_LSIZE, %g5
 132 
 133         /*
 134          * Use a different flush address to avoid recursion if the error
 135          * exists in ecache_flushaddr.
 136          */
 137         ASM_LDX(%g6, ecache_tl1_flushaddr)
 138         cmp     %g6, -1         ! check if address is valid
 139         be      %xcc, fecc_tl1_err
 140           nop
 141         CH_ECACHE_FLUSHALL(%g4, %g5, %g6)
 142 
 143         /*
 144          * Restore CEEN and NCEEN to the previous state.
 145          */
 146         stxa    %g7, [%g0]ASI_ESTATE_ERR
 147         membar  #Sync
 148 
 149         /*
 150          * If we turned off the D$, then flush it and turn it back on.
 151          */
 152         ldxa    [%g1 + CH_ERR_TL1_TMP]%asi, %g3
 153         andcc   %g3, CH_ERR_TSTATE_DC_ON, %g0
 154         bz      %xcc, 3f
 155           nop
 156 
 157         /*
 158          * Flush the D$.
 159          */
 160         ASM_LD(%g4, dcache_size)
 161         ASM_LD(%g5, dcache_linesize)
 162         CH_DCACHE_FLUSHALL(%g4, %g5, %g6)
 163 
 164         /*
 165          * Turn the D$ back on.
 166          */
 167         ldxa    [%g0]ASI_DCU, %g3
 168         or      %g3, DCU_DC, %g3
 169         stxa    %g3, [%g0]ASI_DCU
 170         membar  #Sync
 171 3:
 172         /*
 173          * If we turned off the I$, then flush it and turn it back on.
 174          */
 175         ldxa    [%g1 + CH_ERR_TL1_TMP]%asi, %g3
 176         andcc   %g3, CH_ERR_TSTATE_IC_ON, %g0
 177         bz      %xcc, 4f
 178           nop
 179 
 180         /*
 181          * Flush the I$.
 182          */
 183         ASM_LD(%g4, icache_size)
 184         ASM_LD(%g5, icache_linesize)
 185         CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3)
 186 
 187         /*
 188          * Turn the I$ back on.  Changing DCU_IC requires flush.
 189          */
 190         ldxa    [%g0]ASI_DCU, %g3
 191         or      %g3, DCU_IC, %g3
 192         stxa    %g3, [%g0]ASI_DCU
 193         flush   %g0     
 194 4:
 195 
 196 #ifdef TRAPTRACE
 197         /*
 198          * Get current trap trace entry physical pointer.
 199          */
 200         CPU_INDEX(%g6, %g5)
 201         sll     %g6, TRAPTR_SIZE_SHIFT, %g6
 202         set     trap_trace_ctl, %g5
 203         add     %g6, %g5, %g6
 204         ld      [%g6 + TRAPTR_LIMIT], %g5
 205         tst     %g5
 206         be      %icc, skip_traptrace
 207           nop
 208         ldx     [%g6 + TRAPTR_PBASE], %g5
 209         ld      [%g6 + TRAPTR_OFFSET], %g4
 210         add     %g5, %g4, %g5
 211 
 212         /*
 213          * Create trap trace entry.
 214          */
 215         rd      %asi, %g7
 216         wr      %g0, TRAPTR_ASI, %asi
 217         rd      STICK, %g4
 218         stxa    %g4, [%g5 + TRAP_ENT_TICK]%asi
 219         rdpr    %tl, %g4
 220         stha    %g4, [%g5 + TRAP_ENT_TL]%asi
 221         rdpr    %tt, %g4
 222         stha    %g4, [%g5 + TRAP_ENT_TT]%asi
 223         rdpr    %tpc, %g4
 224         stna    %g4, [%g5 + TRAP_ENT_TPC]%asi
 225         rdpr    %tstate, %g4
 226         stxa    %g4, [%g5 + TRAP_ENT_TSTATE]%asi
 227         stna    %sp, [%g5 + TRAP_ENT_SP]%asi
 228         stna    %g0, [%g5 + TRAP_ENT_TR]%asi
 229         wr      %g0, %g7, %asi
 230         ldxa    [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3
 231         ldxa    [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4
 232         wr      %g0, TRAPTR_ASI, %asi
 233         stna    %g3, [%g5 + TRAP_ENT_F1]%asi
 234         stna    %g4, [%g5 + TRAP_ENT_F2]%asi
 235         wr      %g0, %g7, %asi
 236         ldxa    [%g1 + CH_ERR_TL1_AFAR]%asi, %g3
 237         ldxa    [%g1 + CH_ERR_TL1_AFSR]%asi, %g4
 238         wr      %g0, TRAPTR_ASI, %asi
 239         stna    %g3, [%g5 + TRAP_ENT_F3]%asi
 240         stna    %g4, [%g5 + TRAP_ENT_F4]%asi
 241         wr      %g0, %g7, %asi
 242 
 243         /*
 244          * Advance trap trace pointer.
 245          */
 246         ld      [%g6 + TRAPTR_OFFSET], %g5
 247         ld      [%g6 + TRAPTR_LIMIT], %g4
 248         st      %g5, [%g6 + TRAPTR_LAST_OFFSET]
 249         add     %g5, TRAP_ENT_SIZE, %g5
 250         sub     %g4, TRAP_ENT_SIZE, %g4
 251         cmp     %g5, %g4
 252         movge   %icc, 0, %g5
 253         st      %g5, [%g6 + TRAPTR_OFFSET]
 254 skip_traptrace:
 255 #endif  /* TRAPTRACE */
 256 
 257         /*
 258          * If nesting count is not zero, skip all the AFSR/AFAR
 259          * handling and just do the necessary cache-flushing.
 260          */
 261         ldxa    [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2
 262         brnz    %g2, 6f
 263           nop
 264 
 265         /*
 266          * If a UCU followed by a WDU has occurred go ahead and panic
 267          * since a UE will occur (on the retry) before the UCU and WDU
 268          * messages are enqueued.
 269          */
 270         ldxa    [%g1 + CH_ERR_TL1_AFSR]%asi, %g3
 271         set     1, %g4
 272         sllx    %g4, C_AFSR_UCU_SHIFT, %g4
 273         btst    %g4, %g3                ! UCU in original AFSR?
 274         bz      %xcc, 6f
 275           nop
 276         ldxa    [%g0]ASI_AFSR, %g4      ! current AFSR
 277         or      %g3, %g4, %g3           ! %g3 = original + current AFSR
 278         set     1, %g4
 279         sllx    %g4, C_AFSR_WDU_SHIFT, %g4
 280         btst    %g4, %g3                ! WDU in original or current AFSR?
 281         bnz     %xcc, fecc_tl1_err
 282           nop
 283 
 284 6:
 285         /*
 286          * We fall into this macro if we've successfully logged the error in
 287          * the ch_err_tl1_data structure and want the PIL15 softint to pick
 288          * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
 289          * Restores the %g registers and issues retry.
 290          */
 291         CH_ERR_TL1_EXIT;
 292 
 293         /*
 294          * Establish panic exit label.
 295          */
 296         CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
 297 
 298         SET_SIZE(fast_ecc_tl1_err)
 299 
 300 
 301         ENTRY(scrubphys)
 302         rdpr    %pstate, %o4
 303         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 304         wrpr    %o5, %g0, %pstate       ! clear IE, AM bits
 305 
 306         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 307         casxa   [%o0]ASI_MEM, %g0, %g0
 308         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 309 
 310         wrpr    %g0, %o4, %pstate       ! restore earlier pstate register value
 311 
 312         retl
 313         membar  #Sync                   ! move the data out of the load buffer
 314         SET_SIZE(scrubphys)
 315 
 316 
 317         ENTRY(clearphys)
 318         /* turn off IE, AM bits */
 319         rdpr    %pstate, %o4
 320         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 321         wrpr    %o5, %g0, %pstate
 322 
 323         /* turn off NCEEN */
 324         ldxa    [%g0]ASI_ESTATE_ERR, %o5
 325         andn    %o5, EN_REG_NCEEN, %o3
 326         stxa    %o3, [%g0]ASI_ESTATE_ERR
 327         membar  #Sync
 328 
 329         /* align address passed with 64 bytes subblock size */
 330         mov     CH_ECACHE_SUBBLK_SIZE, %o2
 331         andn    %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
 332 
 333         /* move the good data into the W$ */
 334 1:
 335         subcc   %o2, 8, %o2
 336         ldxa    [%g1 + %o2]ASI_MEM, %g2
 337         bge     1b
 338           stxa  %g2, [%g1 + %o2]ASI_MEM
 339 
 340         /* now overwrite the bad data */
 341         setx    0xbadecc00badecc01, %g1, %g2
 342         stxa    %g2, [%o0]ASI_MEM
 343         mov     8, %g1
 344         stxa    %g2, [%o0 + %g1]ASI_MEM
 345         
 346         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 347         casxa   [%o0]ASI_MEM, %g0, %g0
 348         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 349 
 350         /* clear the AFSR */
 351         ldxa    [%g0]ASI_AFSR, %o1
 352         stxa    %o1, [%g0]ASI_AFSR
 353         membar  #Sync
 354 
 355         /* turn NCEEN back on */
 356         stxa    %o5, [%g0]ASI_ESTATE_ERR
 357         membar  #Sync
 358 
 359         /* return and re-enable IE and AM */
 360         retl
 361           wrpr  %g0, %o4, %pstate
 362         SET_SIZE(clearphys)
 363 
 364 
 365         ENTRY(ecache_flush_line)
 366 
 367         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 368 
 369         retl
 370           nop
 371         SET_SIZE(ecache_flush_line)
 372 
 373 
 374         ENTRY(flush_ipb)
 375         retl
 376         nop
 377         SET_SIZE(flush_ipb)
 378