1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Assembly code support for the jalapeno module
  27  */
  28 
  29 #include "assym.h"
  30 
  31 #include <sys/asm_linkage.h>
  32 #include <sys/mmu.h>
  33 #include <vm/hat_sfmmu.h>
  34 #include <sys/machparam.h>
  35 #include <sys/machcpuvar.h>
  36 #include <sys/machthread.h>
  37 #include <sys/machtrap.h>
  38 #include <sys/privregs.h>
  39 #include <sys/asm_linkage.h>
  40 #include <sys/trap.h>
  41 #include <sys/cheetahregs.h>
  42 #include <sys/us3_module.h>
  43 #include <sys/xc_impl.h>
  44 #include <sys/intreg.h>
  45 #include <sys/async.h>
  46 #include <sys/clock.h>
  47 #include <sys/cheetahasm.h>
  48 
  49 #ifdef TRAPTRACE
  50 #include <sys/traptrace.h>
  51 #endif /* TRAPTRACE */
  52 
  53 /* BEGIN CSTYLED */
  54 
  55 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
  56 
  57 #define CHK_JP_ERRATA85_ENABLED(scr, label)                             \
  58         ASM_LD(scr, jp_errata_85_active);                               \
  59         cmp     scr, 1;                                                 \
  60         bne     %icc, label;                                            \
  61         nop
  62 
  63 #define SET_64BIT_PA(dest, scr, hi32, lo32)                             \
  64         set     hi32, scr;                                              \
  65         sllx    scr, 32, scr;                                           \
  66         sethi   %hi(lo32), dest;                                        \
  67         or      dest, %lo(lo32), dest;                                  \
  68         or      scr, dest, dest
  69 
  70 /*
  71  * Macro to trigger Jalapeno/Tomatillo speed change 
  72  *   j_chng_pa - scratch register
  73  *   scr - scratch register
  74  */
  75 #define JP_ESTAR_TRIGGER(j_chng_pa, scr)                                \
  76         SET_64BIT_PA(j_chng_pa, scr, TOM_HIGH_PA, M_T_J_CHNG_INIT_PA);  \
  77         ldxa    [j_chng_pa]ASI_IO, scr;                                 \
  78 5:                                                                      \
  79         and     scr, TOM_TRIGGER_MASK, scr;                             \
  80         cmp     scr, TOM_TRIGGER;                                       \
  81         be,pt %icc, 5b;                 /* wait while 10 */             \
  82         ldxa    [j_chng_pa]ASI_IO, scr;                                 \
  83         andn    scr, TOM_TRIGGER_MASK, scr;                             \
  84         stxa    scr, [j_chng_pa]ASI_IO; /* clear j_chng[1:0] */         \
  85         or      scr, TOM_TRIGGER, scr;                                  \
  86         stxa    scr, [j_chng_pa]ASI_IO; /* trigger j_chng */            \
  87         ldxa    [j_chng_pa]ASI_IO, scr;                                 \
  88 6:                                                                      \
  89         and     scr, TOM_TRIGGER_MASK, scr;                             \
  90         cmp     scr, TOM_TRIGGER;                                       \
  91         be,pt %icc, 6b;                 /* wait while 10 */             \
  92         ldxa    [j_chng_pa]ASI_IO, scr;                                 \
  93         andn    scr, TOM_TRIGGER_MASK, scr;                             \
  94         stxa    scr, [j_chng_pa]ASI_IO; /* deassert j_chng */
  95 
  96 /*
  97  * Macro to set Jalapeno CPU speed
  98  *   speed - new speed constant
  99  *   scr1  - scratch register
 100  *   scr2  - scratch register
 101  */
 102 #define SET_JP_SPEED(speed, scr1, scr2)                                 \
 103         ldxa    [%g0]ASI_JBUS_CONFIG, scr1;                             \
 104         set     JBUS_CONFIG_ECLK_MASK, scr2;                            \
 105         andn    scr1, scr2, scr1;                                       \
 106         set     speed, scr2;                                            \
 107         or      scr1, scr2, scr1;                                       \
 108         stxa    scr1, [%g0]ASI_JBUS_CONFIG;
 109 
 110 /*
 111  * macro to set Master Tomatillo speed
 112  *   speed - tomatillo speed constant
 113  *   tpa   - tomatillo estar control register PA
 114  *   scr  - scratch register
 115  */
 116 #define SET_TOM_SPEED(speed, tpa, scr)                                  \
 117         ldxa    [tpa]ASI_IO, scr;                                       \
 118         andn    scr, TOM_ESTAR_ELCK_MASK, scr;                          \
 119         or      scr, speed, scr;                                        \
 120         stxa    scr, [tpa]ASI_IO;
 121 
 122 /*
 123  * macro to check and set Slave Tomatillo speed
 124  *   speed - tomatillo speed constant
 125  *   scr1   - scratch register
 126  *   scr2   - scratch register
 127  */
 128 
 129 #define SET_SLAVE_T_SPEED(speed, scr1, scr2)                            \
 130         ldxa    [%g0]ASI_JBUS_CONFIG, scr2;                             \
 131         srlx    scr2, JBUS_SLAVE_T_PORT_BIT, scr2;                      \
 132         btst    1, scr2;                                                \
 133         bz,pt   %icc, 4f;                                               \
 134         nop;                                                            \
 135         SET_64BIT_PA(scr1, scr2, TOM_HIGH_PA, S_T_ESTAR_CTRL_PA);       \
 136         SET_TOM_SPEED(speed, scr1, scr2);                               \
 137 4:
 138 
 139 
 140 /*
 141  * macro to adjust ASI_MCU_CTL_REG1[26:25] fsm bits according to
 142  * new cpu speed: fsm[1:0]=11b for full speed, fsm[1:0]=0 for estar speed
 143  *    value - fsm bit value constant
 144  *    scr1  - scratch register
 145  *    scr2  - scratch register
 146  */
 147 #define JP_ADJUST_FSM(value, scr1, scr2)                                \
 148         ldxa    [%g0]ASI_MCU_CTRL, scr1;                                \
 149         set     JP_MCU_FSM_MASK, scr2;                                  \
 150         andn    scr1, scr2, scr1;                                       \
 151         set     value, scr2;                                            \
 152         or      scr1, scr2, scr1;                                       \
 153         stxa    scr1, [%g0]ASI_MCU_CTRL;                                \
 154         membar  #Sync;
 155 
 156 /*
 157  * JP_FORCE_FULL_SPEED and its fellow macros are for Jalapeno
 158  * workstation to work around Errata 85. The front portion of
 159  * it packs JP speed(14..13) and Tomatillo speed(5..0) into one
 160  * register.
 161  *
 162  * Current code assumes that these two fields are non-overlapping.
 163  * If that assumption changes, then this code won't work. If so, we
 164  * force a compile time error by not defining the JP_FORCE_FULL_SPEED
 165  * and JP_RESTORE_SPEED macros below.
 166  */
 167 
 168 #if !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK)
 169 
 170 /*
 171  * Macro to force Jalapeno/Tomatillo to full speed
 172  *   old_lvl - register used to save original cpu, tomatillo speed 
 173  *   scr2 - scratch register
 174  *   scr3 - scratch register
 175  *   scr4 - scratch register
 176  */
 177 #define JP_FORCE_FULL_SPEED(old_lvl, scr2, scr3, scr4)                  \
 178         ldxa    [%g0]ASI_JBUS_CONFIG, old_lvl;                          \
 179         set     JBUS_CONFIG_ECLK_MASK, scr4;                            \
 180         and     old_lvl, scr4, old_lvl;                                 \
 181         SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA);       \
 182         ldxa    [scr2]ASI_IO, scr3;                                     \
 183         set     TOM_ESTAR_ELCK_MASK, scr4;                              \
 184         and     scr3, scr4, scr3;                                       \
 185         or      old_lvl, scr3, old_lvl;                                 \
 186         /* original jp and tomatillo speed saved in old_lvl */          \
 187                                                                         \
 188         /* either intended or currently at full speed */                \
 189         set     JBUS_CONFIG_ECLK_MASK, scr4;                            \
 190         andcc   old_lvl, scr4, %g0;                                     \
 191         bz,pt   %icc, 8f;                                               \
 192         nop;                                                            \
 193         /* go through 1/2 speed. */                                     \
 194         SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4);                   \
 195         SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3);                      \
 196         SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4);                  \
 197         JP_ADJUST_FSM(0, scr3, scr4);                                   \
 198         set     jp_estar_tl0_data, scr3;                                \
 199         ldx     [scr3], %g0;                                            \
 200         membar  #Sync;          /* or busy wait 1us */                  \
 201         JP_ESTAR_TRIGGER(scr3, scr4);                                   \
 202 8:                                                                      \
 203         /* bring to 1:1 speed */                                        \
 204         SET_JP_SPEED(JBUS_CONFIG_ECLK_1, scr3, scr4);                   \
 205         SET_TOM_SPEED(TOM_FULL_SPEED, scr2, scr3);                      \
 206         SET_SLAVE_T_SPEED(TOM_FULL_SPEED, scr3, scr4);                  \
 207         JP_ADJUST_FSM(JP_MCU_FSM_MASK, scr3, scr4);                     \
 208         JP_ESTAR_TRIGGER(scr3, scr4)
 209 
 210 
 211 /*
 212  * Macro to restore Jalapeno/Tomatillo to original speed
 213  *     old_lvl - register contains saved original cpu, tomatillo speed 
 214  *     scr2 - scratch register
 215  *     scr3 - scratch register
 216  *     scr4 - scratch register
 217  *
 218  * If trap had occured in the middle of ppm cpu speed transtion, then
 219  * old_lvl[31:10] contains the intended new speed written into jbus_config.
 220  * if old_lvl[9:0] is inconsistent with old_lvl[31:10], then the trap surely
 221  * interrupted the ppm cpu speed transition, otherwise nothing for sure.  
 222  * We'll restore the intended/then-current speed, that should cause no
 223  * trouble to subsequent ppm cpu speed change code.
 224  */
 225 #define JP_RESTORE_SPEED(old_lvl, scr2, scr3, scr4)                     \
 226         srlx    old_lvl, JBUS_CONFIG_ECLK_SHIFT, scr2;                  \
 227         and     scr2, 3, scr2;                                          \
 228         add     scr2, 1, scr2;                                          \
 229         cmp     scr2, 3;                                                \
 230         bne,pt  %icc, 7f;                                               \
 231           nop;                                                          \
 232         set     TOM_SLOW_SPEED, scr2;                                   \
 233         /* scr2 contains tom speed according to intended jp speed */    \
 234 7:                                                                      \
 235         andn    old_lvl, TOM_ESTAR_ELCK_MASK, old_lvl;                  \
 236         or      scr2, old_lvl, old_lvl;                                 \
 237         /* updated old_lvl to contain intended jp and tom speed */      \
 238         andcc   old_lvl, TOM_FULL_SPEED, %g0;                           \
 239         bnz,pt  %icc, 9f;       /* intended full, already at full */    \
 240         nop;                                                            \
 241                                                                         \
 242         /* go to half speed */                                          \
 243         SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4);                   \
 244         SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA);       \
 245         SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3);                      \
 246         SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4);                  \
 247         JP_ADJUST_FSM(0, scr3, scr4);                                   \
 248         set     jp_estar_tl0_data, scr3;                                \
 249         ldx     [scr3], %g0;                                            \
 250         membar  #Sync;                                                  \
 251         JP_ESTAR_TRIGGER(scr3, scr4);                                   \
 252         andcc   old_lvl, TOM_SLOW_SPEED, %g0;                           \
 253         bz,pt   %icc, 9f;       /* intended 1:2, already at 1:2 */      \
 254           nop;                                                          \
 255                                                                         \
 256         /* go to 1:32 speed */                                          \
 257         SET_JP_SPEED(JBUS_CONFIG_ECLK_32, scr3, scr4);                  \
 258         SET_TOM_SPEED(TOM_SLOW_SPEED, scr2, scr3);                      \
 259         SET_SLAVE_T_SPEED(TOM_SLOW_SPEED, scr3, scr4);                  \
 260         JP_ESTAR_TRIGGER(scr3, scr4);                                   \
 261 9:
 262 
 263 #endif /* !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) */
 264 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 265 
 266 /*
 267  * Jalapeno version to reflush an Ecache line by index.
 268  * Will flush all 4 ways (with only one scratch register).
 269  * Note that the code will be faster if we use 2 scratch registers.
 270  */
 271 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr1)                   \
 272         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 273         ldxa    [index]ASI_EC_DIAG, %g0;                                \
 274         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 275         mov     1, scr1;                                                \
 276         sllx    scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1;                    \
 277         add     scr1, index, scr1;                                      \
 278         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 279         ldxa    [scr1]ASI_EC_DIAG, %g0;                                 \
 280         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 281         mov     2, scr1;                                                \
 282         sllx    scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1;                    \
 283         add     scr1, index, scr1;                                      \
 284         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 285         ldxa    [scr1]ASI_EC_DIAG, %g0;                                 \
 286         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 287         mov     3, scr1;                                                \
 288         sllx    scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1;                    \
 289         add     scr1, index, scr1;                                      \
 290         JP_EC_DIAG_ACCESS_MEMBAR;                                       \
 291         ldxa    [scr1]ASI_EC_DIAG, %g0;                                 \
 292         JP_EC_DIAG_ACCESS_MEMBAR
 293 
 294 /*
 295  * Jalapeno version of ecache_flush_line.  Uses Jalapeno Ecache Displacement
 296  * Flush feature to flush all 4 sets/ways.
 297  */
 298 #define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2)            \
 299         CPU_INDEX(scr1, scr2);                                          \
 300         sllx    scr1, JP_ECFLUSH_PORTID_SHIFT, scr1;                    \
 301         set     JP_ECACHE_IDX_DISP_FLUSH, scr2;                         \
 302         or      scr2, scr1, scr2;                                       \
 303         sub     ec_set_size, 1, scr1;                                   \
 304         and     physaddr, scr1, scr1;                                   \
 305         or      scr2, scr1, scr1;                                       \
 306         ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2)
 307 
 308 /*
 309  * Macro for getting ecache size from cpunodes structure
 310  *  scr1:    Scratch, ecache size returned in this
 311  *  scr2:    Scratch
 312  */
 313 #define GET_ECACHE_SIZE(scr1, scr2)                                     \
 314         CPU_INDEX(scr1, scr2);                                          \
 315         mulx    scr1, CPU_NODE_SIZE, scr1;                              \
 316         set     cpunodes + ECACHE_SIZE, scr2;                           \
 317         ld      [scr1 + scr2], scr1
 318 
 319 /* END CSTYLED */
 320 
 321 /*
 322  * Ship mondo to aid using implicit busy/nack pair (bn ignored)
 323  */
 324         ENTRY_NP(shipit)
 325         sll     %o0, IDCR_PID_SHIFT, %g1        ! IDCR<18:14> = agent id
 326         or      %g1, IDCR_OFFSET, %g1           ! IDCR<13:0> = 0x70
 327         stxa    %g0, [%g1]ASI_INTR_DISPATCH     ! interrupt vector dispatch
 328         membar  #Sync
 329         retl
 330         nop
 331         SET_SIZE(shipit)
 332 
 333 
 334 /*
 335  * flush_ecache:
 336  *      %o0 - 64 bit physical address
 337  *      %o1 - ecache size
 338  *      %o2 - ecache linesize
 339  */
 340 
 341         ENTRY(flush_ecache)
 342 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 343         CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_1);
 344         JP_FORCE_FULL_SPEED(%o3, %g1, %g2, %g3);        /* %o3: saved speed */
 345 flush_ecache_1:
 346 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 347 
 348         /*
 349          * Flush the entire Ecache using displacement flush.
 350          */
 351         ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)
 352 
 353 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 354         CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_2);
 355         JP_RESTORE_SPEED(%o3, %g1, %g2, %g3);           /* %o3: saved speed */
 356 flush_ecache_2:
 357 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 358 
 359         retl
 360         nop
 361         SET_SIZE(flush_ecache)
 362 
 363 
 364         .section ".text"
 365         .align  64
 366         ENTRY_NP(fast_ecc_err)
 367         
 368         /*
 369          * Turn off CEEN and NCEEN.
 370          */
 371         ldxa    [%g0]ASI_ESTATE_ERR, %g3
 372         andn    %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
 373         stxa    %g4, [%g0]ASI_ESTATE_ERR
 374         membar  #Sync                   ! membar sync required
 375 
 376         /*
 377          * Do the CPU log out capture.
 378          *   %g3 = "failed?" return value.
 379          *   %g2 = Input = AFAR. Output the clo_flags info which is passed
 380          *         into this macro via %g4. Output only valid if cpu_private
 381          *         struct has not been initialized.
 382          *   CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
 383          *   %g4 = Trap information stored in the cpu logout flags field
 384          *   %g5 = scr1
 385          *   %g6 = scr2
 386          *   %g3 = scr3
 387          *   %g4 = scr4
 388          */
 389         and     %g3, EN_REG_CEEN, %g4           ! store the CEEN value, TL=0
 390         set     CHPR_FECCTL0_LOGOUT, %g6
 391         DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
 392 
 393         /*
 394          * Flush the Ecache to get the error out of the Ecache.  If the UCC
 395          * or UCU is on a dirty line, then the following flush will turn
 396          * that into a WDC or WDU, respectively.
 397          */
 398         CPU_INDEX(%g4, %g5)
 399         mulx    %g4, CPU_NODE_SIZE, %g4
 400         set     cpunodes, %g5
 401         add     %g4, %g5, %g4
 402         ld      [%g4 + ECACHE_LINESIZE], %g5
 403         ld      [%g4 + ECACHE_SIZE], %g4
 404 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 405         CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_1);
 406         set     jp_estar_tl0_data, %g6
 407         stx     %g2, [%g6 + 0]
 408         stx     %g3, [%g6 + 8]
 409         JP_FORCE_FULL_SPEED(%g2, %g3, %g6, %g7)         /* %g2: saved speed */
 410 fast_ecc_err_1:
 411 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 412         ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)
 413 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 414         CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_2);
 415         JP_RESTORE_SPEED(%g2, %g3, %g6, %g7)            /* %g2: saved speed */
 416         set     jp_estar_tl0_data, %g6
 417         ldx     [%g6 + 0], %g2
 418         ldx     [%g6 + 8], %g3
 419 fast_ecc_err_2:
 420 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 421 
 422         /*
 423          * Flush the Dcache.  Since bad data could have been installed in
 424          * the Dcache we must flush it before re-enabling it.
 425          */
 426         ASM_LD(%g5, dcache_size)
 427         ASM_LD(%g6, dcache_linesize)
 428         CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
 429 
 430         /*
 431          * Flush the Icache.  Since we turned off the Icache to capture the
 432          * Icache line it is now stale or corrupted and we must flush it
 433          * before re-enabling it.
 434          */
 435         GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_4);
 436         ld      [%g5 + CHPR_ICACHE_LINESIZE], %g6
 437         ba,pt   %icc, 5f
 438           ld    [%g5 + CHPR_ICACHE_SIZE], %g5
 439 fast_ecc_err_4:
 440         ASM_LD(%g5, icache_size)
 441         ASM_LD(%g6, icache_linesize)
 442 5:
 443         CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
 444 
 445         /*
 446          * Restore the Dcache and Icache to the previous state.
 447          */
 448         stxa    %g1, [%g0]ASI_DCU
 449         flush   %g0     /* flush required after changing the IC bit */
 450 
 451         /*
 452          * Make sure our CPU logout operation was successful.
 453          */
 454         cmp     %g3, %g0
 455         be      8f
 456           nop
 457 
 458         /*
 459          * If the logout structure had been busy, how many times have
 460          * we tried to use it and failed (nesting count)? If we have
 461          * already recursed a substantial number of times, then we can
 462          * assume things are not going to get better by themselves and
 463          * so it would be best to panic.
 464          */
 465         cmp     %g3, CLO_NESTING_MAX
 466         blt     7f
 467           nop
 468 
 469         call ptl1_panic
 470           mov   PTL1_BAD_ECC, %g1
 471 
 472 7:
 473         /*
 474          * Otherwise, if the logout structure was busy but we have not
 475          * nested more times than our maximum value, then we simply
 476          * issue a retry. Our TL=0 trap handler code will check and
 477          * clear the AFSR after it is done logging what is currently
 478          * in the logout struct and handle this event at that time.
 479          */
 480         retry
 481 8:
 482         /*
 483          * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
 484          * already at PIL 15.
 485          */
 486         set     cpu_fast_ecc_error, %g1
 487         rdpr    %pil, %g4
 488         cmp     %g4, PIL_14
 489         ba      sys_trap
 490           movl  %icc, PIL_14, %g4
 491 
 492         SET_SIZE(fast_ecc_err)
 493 
 494 
 495 /*
 496  * Fast ECC error at TL>0 handler
 497  * We get here via trap 70 at TL>0->Software trap 0 at TL>0.  We enter
 498  * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
 499  * For a complete description of the Fast ECC at TL>0 handling see the
 500  * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
 501  * us3_common_asm.s
 502  */
 503 
 504         .section ".text"
 505         .align  64
 506         ENTRY_NP(fast_ecc_tl1_err)
 507 
 508         /*
 509          * This macro turns off the D$/I$ if they are on and saves their
 510          * original state in ch_err_tl1_tmp, saves all the %g registers in the
 511          * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
 512          * the %tpc in ch_err_tl1_tpc.  At the end of this macro, %g1 will
 513          * point to the ch_err_tl1_data structure and the original D$/I$ state
 514          * will be saved in ch_err_tl1_tmp.  All %g registers except for %g1
 515          * will be available.
 516          */
 517         CH_ERR_TL1_FECC_ENTER;
 518 
 519         /*
 520          * Get the diagnostic logout data.  %g4 must be initialized to
 521          * current CEEN state, %g5 must point to logout structure in
 522          * ch_err_tl1_data_t.  %g3 will contain the nesting count upon
 523          * return.
 524          */
 525         ldxa    [%g0]ASI_ESTATE_ERR, %g4
 526         and     %g4, EN_REG_CEEN, %g4
 527         add     %g1, CH_ERR_TL1_LOGOUT, %g5
 528         DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4)
 529 
 530         /*
 531          * If the logout nesting count is exceeded, we're probably
 532          * not making any progress, try to panic instead.
 533          */
 534         cmp     %g3, CLO_NESTING_MAX
 535         bge     fecc_tl1_err
 536           nop
 537 
 538         /*
 539          * Save the current CEEN and NCEEN state in %g7 and turn them off
 540          * before flushing the Ecache.
 541          */
 542         ldxa    [%g0]ASI_ESTATE_ERR, %g7
 543         andn    %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5
 544         stxa    %g5, [%g0]ASI_ESTATE_ERR
 545         membar  #Sync
 546 
 547         /*
 548          * Flush the Ecache, using the largest possible cache size with the
 549          * smallest possible line size since we can't get the actual sizes
 550          * from the cpu_node due to DTLB misses.
 551          */
 552         set     JP_ECACHE_MAX_SIZE, %g4
 553 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 554         CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_1);
 555         set     jp_estar_tl1_data, %g6
 556         stx     %g2, [%g6 + 0]
 557         stx     %g3, [%g6 + 8]
 558         JP_FORCE_FULL_SPEED(%g2, %g3, %g5, %g6)
 559 fast_ecc_tl1_err_1:
 560 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 561         ECACHE_FLUSHALL(%g4, JP_ECACHE_MAX_LSIZE, %g5, %g6)
 562 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 563         CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_2);
 564         JP_RESTORE_SPEED(%g2, %g3, %g5, %g6)
 565         set     jp_estar_tl1_data, %g6
 566         ldx     [%g6 + 0], %g2
 567         ldx     [%g6 + 8], %g3
 568 fast_ecc_tl1_err_2:
 569 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 570 
 571         /*
 572          * Restore CEEN and NCEEN to the previous state.
 573          */
 574         stxa    %g7, [%g0]ASI_ESTATE_ERR
 575         membar  #Sync
 576 
 577         /*
 578          * If we turned off the D$, then flush it and turn it back on.
 579          */
 580         ldxa    [%g1 + CH_ERR_TL1_TMP]%asi, %g3
 581         andcc   %g3, CH_ERR_TSTATE_DC_ON, %g0
 582         bz      %xcc, 3f
 583           nop
 584 
 585         /*
 586          * Flush the D$.
 587          */
 588         ASM_LD(%g4, dcache_size)
 589         ASM_LD(%g5, dcache_linesize)
 590         CH_DCACHE_FLUSHALL(%g4, %g5, %g6)
 591 
 592         /*
 593          * Turn the D$ back on.
 594          */
 595         ldxa    [%g0]ASI_DCU, %g3
 596         or      %g3, DCU_DC, %g3
 597         stxa    %g3, [%g0]ASI_DCU
 598         membar  #Sync
 599 3:
 600         /*
 601          * If we turned off the I$, then flush it and turn it back on.
 602          */
 603         ldxa    [%g1 + CH_ERR_TL1_TMP]%asi, %g3
 604         andcc   %g3, CH_ERR_TSTATE_IC_ON, %g0
 605         bz      %xcc, 4f
 606           nop
 607 
 608         /*
 609          * Flush the I$.
 610          */
 611         ASM_LD(%g4, icache_size)
 612         ASM_LD(%g5, icache_linesize)
 613         CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3)
 614 
 615         /*
 616          * Turn the I$ back on.  Changing DCU_IC requires flush.
 617          */
 618         ldxa    [%g0]ASI_DCU, %g3
 619         or      %g3, DCU_IC, %g3
 620         stxa    %g3, [%g0]ASI_DCU
 621         flush   %g0
 622 4:
 623 
 624 #ifdef TRAPTRACE
 625         /*
 626          * Get current trap trace entry physical pointer.
 627          */
 628         CPU_INDEX(%g6, %g5)
 629         sll     %g6, TRAPTR_SIZE_SHIFT, %g6
 630         set     trap_trace_ctl, %g5
 631         add     %g6, %g5, %g6
 632         ld      [%g6 + TRAPTR_LIMIT], %g5
 633         tst     %g5
 634         be      %icc, skip_traptrace
 635           nop
 636         ldx     [%g6 + TRAPTR_PBASE], %g5
 637         ld      [%g6 + TRAPTR_OFFSET], %g4
 638         add     %g5, %g4, %g5
 639 
 640         /*
 641          * Create trap trace entry.
 642          */
 643         rd      %asi, %g7
 644         wr      %g0, TRAPTR_ASI, %asi
 645         rd      STICK, %g4
 646         stxa    %g4, [%g5 + TRAP_ENT_TICK]%asi
 647         rdpr    %tl, %g4
 648         stha    %g4, [%g5 + TRAP_ENT_TL]%asi
 649         rdpr    %tt, %g4
 650         stha    %g4, [%g5 + TRAP_ENT_TT]%asi
 651         rdpr    %tpc, %g4
 652         stna    %g4, [%g5 + TRAP_ENT_TPC]%asi
 653         rdpr    %tstate, %g4
 654         stxa    %g4, [%g5 + TRAP_ENT_TSTATE]%asi
 655         stna    %sp, [%g5 + TRAP_ENT_SP]%asi
 656         stna    %g0, [%g5 + TRAP_ENT_TR]%asi
 657         wr      %g0, %g7, %asi
 658         ldxa    [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3
 659         ldxa    [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4
 660         wr      %g0, TRAPTR_ASI, %asi
 661         stna    %g3, [%g5 + TRAP_ENT_F1]%asi
 662         stna    %g4, [%g5 + TRAP_ENT_F2]%asi
 663         wr      %g0, %g7, %asi
 664         ldxa    [%g1 + CH_ERR_TL1_AFAR]%asi, %g3
 665         ldxa    [%g1 + CH_ERR_TL1_AFSR]%asi, %g4
 666         wr      %g0, TRAPTR_ASI, %asi
 667         stna    %g3, [%g5 + TRAP_ENT_F3]%asi
 668         stna    %g4, [%g5 + TRAP_ENT_F4]%asi
 669         wr      %g0, %g7, %asi
 670 
 671         /*
 672          * Advance trap trace pointer.
 673          */
 674         ld      [%g6 + TRAPTR_OFFSET], %g5
 675         ld      [%g6 + TRAPTR_LIMIT], %g4
 676         st      %g5, [%g6 + TRAPTR_LAST_OFFSET]
 677         add     %g5, TRAP_ENT_SIZE, %g5
 678         sub     %g4, TRAP_ENT_SIZE, %g4
 679         cmp     %g5, %g4
 680         movge   %icc, 0, %g5
 681         st      %g5, [%g6 + TRAPTR_OFFSET]
 682 skip_traptrace:
 683 #endif  /* TRAPTRACE */
 684 
 685         /*
 686          * If nesting count is not zero, skip all the AFSR/AFAR
 687          * handling and just do the necessary cache-flushing.
 688          */
 689         ldxa    [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2
 690         brnz    %g2, 6f
 691           nop
 692 
 693         /*
 694          * If a UCU followed by a WDU has occurred go ahead and panic
 695          * since a UE will occur (on the retry) before the UCU and WDU
 696          * messages are enqueued.
 697          */
 698         ldxa    [%g1 + CH_ERR_TL1_AFSR]%asi, %g3
 699         set     1, %g4
 700         sllx    %g4, C_AFSR_UCU_SHIFT, %g4
 701         btst    %g4, %g3                ! UCU in original AFSR?
 702         bz      %xcc, 6f
 703           nop
 704         ldxa    [%g0]ASI_AFSR, %g4      ! current AFSR
 705         or      %g3, %g4, %g3           ! %g3 = original + current AFSR
 706         set     1, %g4
 707         sllx    %g4, C_AFSR_WDU_SHIFT, %g4
 708         btst    %g4, %g3                ! WDU in original or current AFSR?
 709         bnz     %xcc, fecc_tl1_err
 710           nop
 711 
 712 6:
 713         /*
 714          * We fall into this macro if we've successfully logged the error in
 715          * the ch_err_tl1_data structure and want the PIL15 softint to pick
 716          * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
 717          * Restores the %g registers and issues retry.
 718          */
 719         CH_ERR_TL1_EXIT;
 720         /*
 721          * Establish panic exit label.
 722          */
 723         CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
 724 
 725         SET_SIZE(fast_ecc_tl1_err)
 726 
 727 
 728         ENTRY(get_jbus_config)
 729         ldxa    [%g0]ASI_JBUS_CONFIG, %o0
 730         retl
 731         nop
 732         SET_SIZE(get_jbus_config)
 733 
 734         ENTRY(set_jbus_config)
 735         stxa    %o0, [%g0]ASI_JBUS_CONFIG
 736         membar  #Sync
 737         retl
 738         nop
 739         SET_SIZE(set_jbus_config)
 740 
 741 
 742         ENTRY(get_mcu_ctl_reg1)
 743         ldxa    [%g0]ASI_MCU_CTRL, %o0  ! MCU control reg1 is at offset 0
 744         retl
 745         nop
 746         SET_SIZE(get_mcu_ctl_reg1)
 747 
 748 
 749         ENTRY(set_mcu_ctl_reg1)
 750         stxa    %o0, [%g0]ASI_MCU_CTRL  ! MCU control reg1 is at offset 0
 751         membar  #Sync
 752         retl
 753         nop
 754         SET_SIZE(set_mcu_ctl_reg1)
 755 
 756 
 757         ENTRY(scrubphys)
 758         rdpr    %pstate, %o4
 759         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 760         wrpr    %o5, %g0, %pstate       ! clear IE, AM bits
 761 
 762 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 763         CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_1);
 764         JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3)         /* %o5: saved speed */
 765 scrubphys_1:
 766 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 767 
 768         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 769         casxa   [%o0]ASI_MEM, %g0, %g0
 770         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 771 
 772 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 773         CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_2);
 774         JP_RESTORE_SPEED(%o5, %g1, %g2, %g3)            /* %o5: saved speed */
 775 scrubphys_2:
 776 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 777 
 778         wrpr    %g0, %o4, %pstate       ! restore earlier pstate register value
 779 
 780         retl
 781         membar  #Sync                   ! move the data out of the load buffer
 782         SET_SIZE(scrubphys)
 783 
 784 
 785         ENTRY(clearphys)
 786         /* turn off IE, AM bits */
 787         rdpr    %pstate, %o4
 788         andn    %o4, PSTATE_IE | PSTATE_AM, %o5
 789         wrpr    %o5, %g0, %pstate
 790 
 791         /* turn off NCEEN */
 792         ldxa    [%g0]ASI_ESTATE_ERR, %o5
 793         andn    %o5, EN_REG_NCEEN, %o3
 794         stxa    %o3, [%g0]ASI_ESTATE_ERR
 795         membar  #Sync
 796 
 797         /* align address passed with 64 bytes subblock size */  
 798         mov     CH_ECACHE_SUBBLK_SIZE, %o2
 799         andn    %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
 800         
 801         /* move the good data into the W$ */    
 802 1:
 803         subcc   %o2, 8, %o2
 804         ldxa    [%g1 + %o2]ASI_MEM, %g2
 805         bge     1b
 806           stxa  %g2, [%g1 + %o2]ASI_MEM
 807 
 808         /* now overwrite the bad data */
 809         setx    0xbadecc00badecc01, %g1, %g2
 810         stxa    %g2, [%o0]ASI_MEM
 811         mov     8, %g1
 812         stxa    %g2, [%o0 + %g1]ASI_MEM
 813 
 814 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 815         CHK_JP_ERRATA85_ENABLED(%g1, clearphys_1);
 816         JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3)         /* %o5: saved speed */
 817 clearphys_1:
 818 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 819 
 820         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 821         casxa   [%o0]ASI_MEM, %g0, %g0
 822         ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
 823 
 824 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 825         CHK_JP_ERRATA85_ENABLED(%g1, clearphys_2);
 826         JP_RESTORE_SPEED(%o5, %g1, %g2, %g3)            /* %o5: saved speed */
 827 clearphys_2:
 828 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 829 
 830         /* clear the AFSR */
 831         ldxa    [%g0]ASI_AFSR, %o1
 832         stxa    %o1, [%g0]ASI_AFSR
 833         membar  #Sync
 834 
 835         /* turn NCEEN back on */
 836         stxa    %o5, [%g0]ASI_ESTATE_ERR
 837         membar  #Sync
 838 
 839         /* return and re-enable IE and AM */
 840         retl
 841           wrpr  %g0, %o4, %pstate
 842         SET_SIZE(clearphys)
 843 
 844 
 845         ENTRY(ecache_flush_line)
 846 
 847 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 848         CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_1);
 849         JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3)         /* %o5: saved speed */
 850 ecache_flush_line_1:
 851 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 852 
 853         ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
 854 
 855 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
 856         CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_2);
 857         JP_RESTORE_SPEED(%o5, %g1, %g2, %g3)            /* %o5: saved speed */
 858 ecache_flush_line_2:
 859 #endif  /* JALAPENO && JALAPENO_ERRATA_85 */
 860 
 861         retl
 862           nop
 863         SET_SIZE(ecache_flush_line)
 864 
 865 
 866 /*
 867  * Perform necessary cpu workaround to ensure jbus ordering.
 868  * Called only from Fire systems.
 869  * CPU's internal "invalidate FIFOs" are flushed.
 870  */
 871 
 872 #define VIS_BLOCKSIZE   64
 873 
 874         .seg    ".data"
 875         .align  VIS_BLOCKSIZE
 876         .type   sync_buf, #object
 877 sync_buf:
 878         .skip   VIS_BLOCKSIZE
 879         .size   sync_buf, VIS_BLOCKSIZE
 880 
 881         ENTRY(jbus_stst_order)
 882         set     sync_buf, %o1
 883 
 884         rd      %fprs, %o2                      ! %o2 = saved fprs
 885         or      %o2, FPRS_FEF, %o3
 886         wr      %g0, %o3, %fprs                 ! make sure fp is enabled
 887         stda    %d0, [%o1]ASI_BLK_COMMIT_P
 888         wr      %o2, 0, %fprs                   ! restore fprs
 889 
 890         retl
 891         membar  #Sync
 892         SET_SIZE(jbus_stst_order)
 893 
 894         ENTRY(flush_ipb)
 895         retl
 896         nop
 897         SET_SIZE(flush_ipb)
 898