1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Assembly code support for the jalapeno module
27 */
28
29 #pragma ident "%Z%%M% %I% %E% SMI"
30
31 #if !defined(lint)
32 #include "assym.h"
33 #endif /* lint */
34
35 #include <sys/asm_linkage.h>
36 #include <sys/mmu.h>
37 #include <vm/hat_sfmmu.h>
38 #include <sys/machparam.h>
39 #include <sys/machcpuvar.h>
40 #include <sys/machthread.h>
41 #include <sys/machtrap.h>
42 #include <sys/privregs.h>
43 #include <sys/asm_linkage.h>
44 #include <sys/trap.h>
45 #include <sys/cheetahregs.h>
46 #include <sys/us3_module.h>
47 #include <sys/xc_impl.h>
48 #include <sys/intreg.h>
49 #include <sys/async.h>
50 #include <sys/clock.h>
51 #include <sys/cheetahasm.h>
52
53 #ifdef TRAPTRACE
54 #include <sys/traptrace.h>
55 #endif /* TRAPTRACE */
56
57 #if !defined(lint)
58
59 /* BEGIN CSTYLED */
60
61 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
62
63 #define CHK_JP_ERRATA85_ENABLED(scr, label) \
64 ASM_LD(scr, jp_errata_85_active); \
65 cmp scr, 1; \
66 bne %icc, label; \
67 nop
68
69 #define SET_64BIT_PA(dest, scr, hi32, lo32) \
70 set hi32, scr; \
71 sllx scr, 32, scr; \
72 sethi %hi(lo32), dest; \
73 or dest, %lo(lo32), dest; \
74 or scr, dest, dest
75
76 /*
77 * Macro to trigger Jalapeno/Tomatillo speed change
78 * j_chng_pa - scratch register
79 * scr - scratch register
80 */
81 #define JP_ESTAR_TRIGGER(j_chng_pa, scr) \
82 SET_64BIT_PA(j_chng_pa, scr, TOM_HIGH_PA, M_T_J_CHNG_INIT_PA); \
83 ldxa [j_chng_pa]ASI_IO, scr; \
84 5: \
85 and scr, TOM_TRIGGER_MASK, scr; \
86 cmp scr, TOM_TRIGGER; \
87 be,pt %icc, 5b; /* wait while 10 */ \
88 ldxa [j_chng_pa]ASI_IO, scr; \
89 andn scr, TOM_TRIGGER_MASK, scr; \
90 stxa scr, [j_chng_pa]ASI_IO; /* clear j_chng[1:0] */ \
91 or scr, TOM_TRIGGER, scr; \
92 stxa scr, [j_chng_pa]ASI_IO; /* trigger j_chng */ \
93 ldxa [j_chng_pa]ASI_IO, scr; \
94 6: \
95 and scr, TOM_TRIGGER_MASK, scr; \
96 cmp scr, TOM_TRIGGER; \
97 be,pt %icc, 6b; /* wait while 10 */ \
98 ldxa [j_chng_pa]ASI_IO, scr; \
99 andn scr, TOM_TRIGGER_MASK, scr; \
100 stxa scr, [j_chng_pa]ASI_IO; /* deassert j_chng */
101
102 /*
103 * Macro to set Jalapeno CPU speed
104 * speed - new speed constant
105 * scr1 - scratch register
106 * scr2 - scratch register
107 */
108 #define SET_JP_SPEED(speed, scr1, scr2) \
109 ldxa [%g0]ASI_JBUS_CONFIG, scr1; \
110 set JBUS_CONFIG_ECLK_MASK, scr2; \
111 andn scr1, scr2, scr1; \
112 set speed, scr2; \
113 or scr1, scr2, scr1; \
114 stxa scr1, [%g0]ASI_JBUS_CONFIG;
115
116 /*
117 * macro to set Master Tomatillo speed
118 * speed - tomatillo speed constant
119 * tpa - tomatillo estar control register PA
120 * scr - scratch register
121 */
122 #define SET_TOM_SPEED(speed, tpa, scr) \
123 ldxa [tpa]ASI_IO, scr; \
124 andn scr, TOM_ESTAR_ELCK_MASK, scr; \
125 or scr, speed, scr; \
126 stxa scr, [tpa]ASI_IO;
127
128 /*
129 * macro to check and set Slave Tomatillo speed
130 * speed - tomatillo speed constant
131 * scr1 - scratch register
132 * scr2 - scratch register
133 */
134
135 #define SET_SLAVE_T_SPEED(speed, scr1, scr2) \
136 ldxa [%g0]ASI_JBUS_CONFIG, scr2; \
137 srlx scr2, JBUS_SLAVE_T_PORT_BIT, scr2; \
138 btst 1, scr2; \
139 bz,pt %icc, 4f; \
140 nop; \
141 SET_64BIT_PA(scr1, scr2, TOM_HIGH_PA, S_T_ESTAR_CTRL_PA); \
142 SET_TOM_SPEED(speed, scr1, scr2); \
143 4:
144
145
146 /*
147 * macro to adjust ASI_MCU_CTL_REG1[26:25] fsm bits according to
148 * new cpu speed: fsm[1:0]=11b for full speed, fsm[1:0]=0 for estar speed
149 * value - fsm bit value constant
150 * scr1 - scratch register
151 * scr2 - scratch register
152 */
153 #define JP_ADJUST_FSM(value, scr1, scr2) \
154 ldxa [%g0]ASI_MCU_CTRL, scr1; \
155 set JP_MCU_FSM_MASK, scr2; \
156 andn scr1, scr2, scr1; \
157 set value, scr2; \
158 or scr1, scr2, scr1; \
159 stxa scr1, [%g0]ASI_MCU_CTRL; \
160 membar #Sync;
161
162 /*
163 * JP_FORCE_FULL_SPEED and its fellow macros are for Jalapeno
164 * workstation to work around Errata 85. The front portion of
165 * it packs JP speed(14..13) and Tomatillo speed(5..0) into one
166 * register.
167 *
168 * Current code assumes that these two fields are non-overlapping.
169 * If that assumption changes, then this code won't work. If so, we
170 * force a compile time error by not defining the JP_FORCE_FULL_SPEED
171 * and JP_RESTORE_SPEED macros below.
172 */
173
174 #if !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK)
175
176 /*
177 * Macro to force Jalapeno/Tomatillo to full speed
178 * old_lvl - register used to save original cpu, tomatillo speed
179 * scr2 - scratch register
180 * scr3 - scratch register
181 * scr4 - scratch register
182 */
183 #define JP_FORCE_FULL_SPEED(old_lvl, scr2, scr3, scr4) \
184 ldxa [%g0]ASI_JBUS_CONFIG, old_lvl; \
185 set JBUS_CONFIG_ECLK_MASK, scr4; \
186 and old_lvl, scr4, old_lvl; \
187 SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA); \
188 ldxa [scr2]ASI_IO, scr3; \
189 set TOM_ESTAR_ELCK_MASK, scr4; \
190 and scr3, scr4, scr3; \
191 or old_lvl, scr3, old_lvl; \
192 /* original jp and tomatillo speed saved in old_lvl */ \
193 \
194 /* either intended or currently at full speed */ \
195 set JBUS_CONFIG_ECLK_MASK, scr4; \
196 andcc old_lvl, scr4, %g0; \
197 bz,pt %icc, 8f; \
198 nop; \
199 /* go through 1/2 speed. */ \
200 SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4); \
201 SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3); \
202 SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4); \
203 JP_ADJUST_FSM(0, scr3, scr4); \
204 set jp_estar_tl0_data, scr3; \
205 ldx [scr3], %g0; \
206 membar #Sync; /* or busy wait 1us */ \
207 JP_ESTAR_TRIGGER(scr3, scr4); \
208 8: \
209 /* bring to 1:1 speed */ \
210 SET_JP_SPEED(JBUS_CONFIG_ECLK_1, scr3, scr4); \
211 SET_TOM_SPEED(TOM_FULL_SPEED, scr2, scr3); \
212 SET_SLAVE_T_SPEED(TOM_FULL_SPEED, scr3, scr4); \
213 JP_ADJUST_FSM(JP_MCU_FSM_MASK, scr3, scr4); \
214 JP_ESTAR_TRIGGER(scr3, scr4)
215
216
217 /*
218 * Macro to restore Jalapeno/Tomatillo to original speed
219 * old_lvl - register contains saved original cpu, tomatillo speed
220 * scr2 - scratch register
221 * scr3 - scratch register
222 * scr4 - scratch register
223 *
224 * If trap had occured in the middle of ppm cpu speed transtion, then
225 * old_lvl[31:10] contains the intended new speed written into jbus_config.
226 * if old_lvl[9:0] is inconsistent with old_lvl[31:10], then the trap surely
227 * interrupted the ppm cpu speed transition, otherwise nothing for sure.
228 * We'll restore the intended/then-current speed, that should cause no
229 * trouble to subsequent ppm cpu speed change code.
230 */
231 #define JP_RESTORE_SPEED(old_lvl, scr2, scr3, scr4) \
232 srlx old_lvl, JBUS_CONFIG_ECLK_SHIFT, scr2; \
233 and scr2, 3, scr2; \
234 add scr2, 1, scr2; \
235 cmp scr2, 3; \
236 bne,pt %icc, 7f; \
237 nop; \
238 set TOM_SLOW_SPEED, scr2; \
239 /* scr2 contains tom speed according to intended jp speed */ \
240 7: \
241 andn old_lvl, TOM_ESTAR_ELCK_MASK, old_lvl; \
242 or scr2, old_lvl, old_lvl; \
243 /* updated old_lvl to contain intended jp and tom speed */ \
244 andcc old_lvl, TOM_FULL_SPEED, %g0; \
245 bnz,pt %icc, 9f; /* intended full, already at full */ \
246 nop; \
247 \
248 /* go to half speed */ \
249 SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4); \
250 SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA); \
251 SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3); \
252 SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4); \
253 JP_ADJUST_FSM(0, scr3, scr4); \
254 set jp_estar_tl0_data, scr3; \
255 ldx [scr3], %g0; \
256 membar #Sync; \
257 JP_ESTAR_TRIGGER(scr3, scr4); \
258 andcc old_lvl, TOM_SLOW_SPEED, %g0; \
259 bz,pt %icc, 9f; /* intended 1:2, already at 1:2 */ \
260 nop; \
261 \
262 /* go to 1:32 speed */ \
263 SET_JP_SPEED(JBUS_CONFIG_ECLK_32, scr3, scr4); \
264 SET_TOM_SPEED(TOM_SLOW_SPEED, scr2, scr3); \
265 SET_SLAVE_T_SPEED(TOM_SLOW_SPEED, scr3, scr4); \
266 JP_ESTAR_TRIGGER(scr3, scr4); \
267 9:
268
269 #endif /* !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) */
270 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
271
272 /*
273 * Jalapeno version to reflush an Ecache line by index.
274 * Will flush all 4 ways (with only one scratch register).
275 * Note that the code will be faster if we use 2 scratch registers.
276 */
277 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr1) \
278 JP_EC_DIAG_ACCESS_MEMBAR; \
279 ldxa [index]ASI_EC_DIAG, %g0; \
280 JP_EC_DIAG_ACCESS_MEMBAR; \
281 mov 1, scr1; \
282 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \
283 add scr1, index, scr1; \
284 JP_EC_DIAG_ACCESS_MEMBAR; \
285 ldxa [scr1]ASI_EC_DIAG, %g0; \
286 JP_EC_DIAG_ACCESS_MEMBAR; \
287 mov 2, scr1; \
288 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \
289 add scr1, index, scr1; \
290 JP_EC_DIAG_ACCESS_MEMBAR; \
291 ldxa [scr1]ASI_EC_DIAG, %g0; \
292 JP_EC_DIAG_ACCESS_MEMBAR; \
293 mov 3, scr1; \
294 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \
295 add scr1, index, scr1; \
296 JP_EC_DIAG_ACCESS_MEMBAR; \
297 ldxa [scr1]ASI_EC_DIAG, %g0; \
298 JP_EC_DIAG_ACCESS_MEMBAR
299
300 /*
301 * Jalapeno version of ecache_flush_line. Uses Jalapeno Ecache Displacement
302 * Flush feature to flush all 4 sets/ways.
303 */
304 #define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2) \
305 CPU_INDEX(scr1, scr2); \
306 sllx scr1, JP_ECFLUSH_PORTID_SHIFT, scr1; \
307 set JP_ECACHE_IDX_DISP_FLUSH, scr2; \
308 or scr2, scr1, scr2; \
309 sub ec_set_size, 1, scr1; \
310 and physaddr, scr1, scr1; \
311 or scr2, scr1, scr1; \
312 ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2)
313
314 /*
315 * Macro for getting ecache size from cpunodes structure
316 * scr1: Scratch, ecache size returned in this
317 * scr2: Scratch
318 */
319 #define GET_ECACHE_SIZE(scr1, scr2) \
320 CPU_INDEX(scr1, scr2); \
321 mulx scr1, CPU_NODE_SIZE, scr1; \
322 set cpunodes + ECACHE_SIZE, scr2; \
323 ld [scr1 + scr2], scr1
324
325 /* END CSTYLED */
326
327 #endif /* !lint */
328
329 #if defined(lint)
330
331 /* ARGSUSED */
332 void
333 shipit(int upaid, int bn)
334 { return; }
335
336 #else /* lint */
337
338 /*
339 * Ship mondo to aid using implicit busy/nack pair (bn ignored)
340 */
341 ENTRY_NP(shipit)
342 sll %o0, IDCR_PID_SHIFT, %g1 ! IDCR<18:14> = agent id
343 or %g1, IDCR_OFFSET, %g1 ! IDCR<13:0> = 0x70
344 stxa %g0, [%g1]ASI_INTR_DISPATCH ! interrupt vector dispatch
345 membar #Sync
346 retl
347 nop
348 SET_SIZE(shipit)
349
350 #endif /* lint */
351
352
353 /*
354 * flush_ecache:
355 * %o0 - 64 bit physical address
356 * %o1 - ecache size
357 * %o2 - ecache linesize
358 */
359 #if defined(lint)
360
361 /*ARGSUSED*/
362 void
363 flush_ecache(uint64_t physaddr, size_t ecache_size, size_t ecache_linesize)
364 {}
365
366 #else /* !lint */
367
368 ENTRY(flush_ecache)
369 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
370 CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_1);
371 JP_FORCE_FULL_SPEED(%o3, %g1, %g2, %g3); /* %o3: saved speed */
372 flush_ecache_1:
373 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
374
375 /*
376 * Flush the entire Ecache using displacement flush.
377 */
378 ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)
379
380 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
381 CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_2);
382 JP_RESTORE_SPEED(%o3, %g1, %g2, %g3); /* %o3: saved speed */
383 flush_ecache_2:
384 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
385
386 retl
387 nop
388 SET_SIZE(flush_ecache)
389
390 #endif /* lint */
391
392
393 #if defined(lint)
394
395 void
396 fast_ecc_err(void)
397 {}
398
399 #else /* lint */
400
401 .section ".text"
402 .align 64
403 ENTRY_NP(fast_ecc_err)
404
405 /*
406 * Turn off CEEN and NCEEN.
407 */
408 ldxa [%g0]ASI_ESTATE_ERR, %g3
409 andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
410 stxa %g4, [%g0]ASI_ESTATE_ERR
411 membar #Sync ! membar sync required
412
413 /*
414 * Do the CPU log out capture.
415 * %g3 = "failed?" return value.
416 * %g2 = Input = AFAR. Output the clo_flags info which is passed
417 * into this macro via %g4. Output only valid if cpu_private
418 * struct has not been initialized.
419 * CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
420 * %g4 = Trap information stored in the cpu logout flags field
421 * %g5 = scr1
422 * %g6 = scr2
423 * %g3 = scr3
424 * %g4 = scr4
425 */
426 and %g3, EN_REG_CEEN, %g4 ! store the CEEN value, TL=0
427 set CHPR_FECCTL0_LOGOUT, %g6
428 DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
429
430 /*
431 * Flush the Ecache to get the error out of the Ecache. If the UCC
432 * or UCU is on a dirty line, then the following flush will turn
433 * that into a WDC or WDU, respectively.
434 */
435 CPU_INDEX(%g4, %g5)
436 mulx %g4, CPU_NODE_SIZE, %g4
437 set cpunodes, %g5
438 add %g4, %g5, %g4
439 ld [%g4 + ECACHE_LINESIZE], %g5
440 ld [%g4 + ECACHE_SIZE], %g4
441 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
442 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_1);
443 set jp_estar_tl0_data, %g6
444 stx %g2, [%g6 + 0]
445 stx %g3, [%g6 + 8]
446 JP_FORCE_FULL_SPEED(%g2, %g3, %g6, %g7) /* %g2: saved speed */
447 fast_ecc_err_1:
448 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
449 ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)
450 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
451 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_2);
452 JP_RESTORE_SPEED(%g2, %g3, %g6, %g7) /* %g2: saved speed */
453 set jp_estar_tl0_data, %g6
454 ldx [%g6 + 0], %g2
455 ldx [%g6 + 8], %g3
456 fast_ecc_err_2:
457 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
458
459 /*
460 * Flush the Dcache. Since bad data could have been installed in
461 * the Dcache we must flush it before re-enabling it.
462 */
463 ASM_LD(%g5, dcache_size)
464 ASM_LD(%g6, dcache_linesize)
465 CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
466
467 /*
468 * Flush the Icache. Since we turned off the Icache to capture the
469 * Icache line it is now stale or corrupted and we must flush it
470 * before re-enabling it.
471 */
472 GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_4);
473 ld [%g5 + CHPR_ICACHE_LINESIZE], %g6
474 ba,pt %icc, 5f
475 ld [%g5 + CHPR_ICACHE_SIZE], %g5
476 fast_ecc_err_4:
477 ASM_LD(%g5, icache_size)
478 ASM_LD(%g6, icache_linesize)
479 5:
480 CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
481
482 /*
483 * Restore the Dcache and Icache to the previous state.
484 */
485 stxa %g1, [%g0]ASI_DCU
486 flush %g0 /* flush required after changing the IC bit */
487
488 /*
489 * Make sure our CPU logout operation was successful.
490 */
491 cmp %g3, %g0
492 be 8f
493 nop
494
495 /*
496 * If the logout structure had been busy, how many times have
497 * we tried to use it and failed (nesting count)? If we have
498 * already recursed a substantial number of times, then we can
499 * assume things are not going to get better by themselves and
500 * so it would be best to panic.
501 */
502 cmp %g3, CLO_NESTING_MAX
503 blt 7f
504 nop
505
506 call ptl1_panic
507 mov PTL1_BAD_ECC, %g1
508
509 7:
510 /*
511 * Otherwise, if the logout structure was busy but we have not
512 * nested more times than our maximum value, then we simply
513 * issue a retry. Our TL=0 trap handler code will check and
514 * clear the AFSR after it is done logging what is currently
515 * in the logout struct and handle this event at that time.
516 */
517 retry
518 8:
519 /*
520 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
521 * already at PIL 15.
522 */
523 set cpu_fast_ecc_error, %g1
524 rdpr %pil, %g4
525 cmp %g4, PIL_14
526 ba sys_trap
527 movl %icc, PIL_14, %g4
528
529 SET_SIZE(fast_ecc_err)
530
531 #endif /* lint */
532
533
534 /*
535 * Fast ECC error at TL>0 handler
536 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter
537 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
538 * For a complete description of the Fast ECC at TL>0 handling see the
539 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
540 * us3_common_asm.s
541 */
542 #if defined(lint)
543
544 void
545 fast_ecc_tl1_err(void)
546 {}
547
548 #else /* lint */
549
550 .section ".text"
551 .align 64
552 ENTRY_NP(fast_ecc_tl1_err)
553
554 /*
555 * This macro turns off the D$/I$ if they are on and saves their
556 * original state in ch_err_tl1_tmp, saves all the %g registers in the
557 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
558 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will
559 * point to the ch_err_tl1_data structure and the original D$/I$ state
560 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1
561 * will be available.
562 */
563 CH_ERR_TL1_FECC_ENTER;
564
565 /*
566 * Get the diagnostic logout data. %g4 must be initialized to
567 * current CEEN state, %g5 must point to logout structure in
568 * ch_err_tl1_data_t. %g3 will contain the nesting count upon
569 * return.
570 */
571 ldxa [%g0]ASI_ESTATE_ERR, %g4
572 and %g4, EN_REG_CEEN, %g4
573 add %g1, CH_ERR_TL1_LOGOUT, %g5
574 DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4)
575
576 /*
577 * If the logout nesting count is exceeded, we're probably
578 * not making any progress, try to panic instead.
579 */
580 cmp %g3, CLO_NESTING_MAX
581 bge fecc_tl1_err
582 nop
583
584 /*
585 * Save the current CEEN and NCEEN state in %g7 and turn them off
586 * before flushing the Ecache.
587 */
588 ldxa [%g0]ASI_ESTATE_ERR, %g7
589 andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5
590 stxa %g5, [%g0]ASI_ESTATE_ERR
591 membar #Sync
592
593 /*
594 * Flush the Ecache, using the largest possible cache size with the
595 * smallest possible line size since we can't get the actual sizes
596 * from the cpu_node due to DTLB misses.
597 */
598 set JP_ECACHE_MAX_SIZE, %g4
599 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
600 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_1);
601 set jp_estar_tl1_data, %g6
602 stx %g2, [%g6 + 0]
603 stx %g3, [%g6 + 8]
604 JP_FORCE_FULL_SPEED(%g2, %g3, %g5, %g6)
605 fast_ecc_tl1_err_1:
606 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
607 ECACHE_FLUSHALL(%g4, JP_ECACHE_MAX_LSIZE, %g5, %g6)
608 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
609 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_2);
610 JP_RESTORE_SPEED(%g2, %g3, %g5, %g6)
611 set jp_estar_tl1_data, %g6
612 ldx [%g6 + 0], %g2
613 ldx [%g6 + 8], %g3
614 fast_ecc_tl1_err_2:
615 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
616
617 /*
618 * Restore CEEN and NCEEN to the previous state.
619 */
620 stxa %g7, [%g0]ASI_ESTATE_ERR
621 membar #Sync
622
623 /*
624 * If we turned off the D$, then flush it and turn it back on.
625 */
626 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3
627 andcc %g3, CH_ERR_TSTATE_DC_ON, %g0
628 bz %xcc, 3f
629 nop
630
631 /*
632 * Flush the D$.
633 */
634 ASM_LD(%g4, dcache_size)
635 ASM_LD(%g5, dcache_linesize)
636 CH_DCACHE_FLUSHALL(%g4, %g5, %g6)
637
638 /*
639 * Turn the D$ back on.
640 */
641 ldxa [%g0]ASI_DCU, %g3
642 or %g3, DCU_DC, %g3
643 stxa %g3, [%g0]ASI_DCU
644 membar #Sync
645 3:
646 /*
647 * If we turned off the I$, then flush it and turn it back on.
648 */
649 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3
650 andcc %g3, CH_ERR_TSTATE_IC_ON, %g0
651 bz %xcc, 4f
652 nop
653
654 /*
655 * Flush the I$.
656 */
657 ASM_LD(%g4, icache_size)
658 ASM_LD(%g5, icache_linesize)
659 CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3)
660
661 /*
662 * Turn the I$ back on. Changing DCU_IC requires flush.
663 */
664 ldxa [%g0]ASI_DCU, %g3
665 or %g3, DCU_IC, %g3
666 stxa %g3, [%g0]ASI_DCU
667 flush %g0
668 4:
669
670 #ifdef TRAPTRACE
671 /*
672 * Get current trap trace entry physical pointer.
673 */
674 CPU_INDEX(%g6, %g5)
675 sll %g6, TRAPTR_SIZE_SHIFT, %g6
676 set trap_trace_ctl, %g5
677 add %g6, %g5, %g6
678 ld [%g6 + TRAPTR_LIMIT], %g5
679 tst %g5
680 be %icc, skip_traptrace
681 nop
682 ldx [%g6 + TRAPTR_PBASE], %g5
683 ld [%g6 + TRAPTR_OFFSET], %g4
684 add %g5, %g4, %g5
685
686 /*
687 * Create trap trace entry.
688 */
689 rd %asi, %g7
690 wr %g0, TRAPTR_ASI, %asi
691 rd STICK, %g4
692 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi
693 rdpr %tl, %g4
694 stha %g4, [%g5 + TRAP_ENT_TL]%asi
695 rdpr %tt, %g4
696 stha %g4, [%g5 + TRAP_ENT_TT]%asi
697 rdpr %tpc, %g4
698 stna %g4, [%g5 + TRAP_ENT_TPC]%asi
699 rdpr %tstate, %g4
700 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi
701 stna %sp, [%g5 + TRAP_ENT_SP]%asi
702 stna %g0, [%g5 + TRAP_ENT_TR]%asi
703 wr %g0, %g7, %asi
704 ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3
705 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4
706 wr %g0, TRAPTR_ASI, %asi
707 stna %g3, [%g5 + TRAP_ENT_F1]%asi
708 stna %g4, [%g5 + TRAP_ENT_F2]%asi
709 wr %g0, %g7, %asi
710 ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3
711 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4
712 wr %g0, TRAPTR_ASI, %asi
713 stna %g3, [%g5 + TRAP_ENT_F3]%asi
714 stna %g4, [%g5 + TRAP_ENT_F4]%asi
715 wr %g0, %g7, %asi
716
717 /*
718 * Advance trap trace pointer.
719 */
720 ld [%g6 + TRAPTR_OFFSET], %g5
721 ld [%g6 + TRAPTR_LIMIT], %g4
722 st %g5, [%g6 + TRAPTR_LAST_OFFSET]
723 add %g5, TRAP_ENT_SIZE, %g5
724 sub %g4, TRAP_ENT_SIZE, %g4
725 cmp %g5, %g4
726 movge %icc, 0, %g5
727 st %g5, [%g6 + TRAPTR_OFFSET]
728 skip_traptrace:
729 #endif /* TRAPTRACE */
730
731 /*
732 * If nesting count is not zero, skip all the AFSR/AFAR
733 * handling and just do the necessary cache-flushing.
734 */
735 ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2
736 brnz %g2, 6f
737 nop
738
739 /*
740 * If a UCU followed by a WDU has occurred go ahead and panic
741 * since a UE will occur (on the retry) before the UCU and WDU
742 * messages are enqueued.
743 */
744 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g3
745 set 1, %g4
746 sllx %g4, C_AFSR_UCU_SHIFT, %g4
747 btst %g4, %g3 ! UCU in original AFSR?
748 bz %xcc, 6f
749 nop
750 ldxa [%g0]ASI_AFSR, %g4 ! current AFSR
751 or %g3, %g4, %g3 ! %g3 = original + current AFSR
752 set 1, %g4
753 sllx %g4, C_AFSR_WDU_SHIFT, %g4
754 btst %g4, %g3 ! WDU in original or current AFSR?
755 bnz %xcc, fecc_tl1_err
756 nop
757
758 6:
759 /*
760 * We fall into this macro if we've successfully logged the error in
761 * the ch_err_tl1_data structure and want the PIL15 softint to pick
762 * it up and log it. %g1 must point to the ch_err_tl1_data structure.
763 * Restores the %g registers and issues retry.
764 */
765 CH_ERR_TL1_EXIT;
766 /*
767 * Establish panic exit label.
768 */
769 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);
770
771 SET_SIZE(fast_ecc_tl1_err)
772
773 #endif /* lint */
774
775
776 #if defined(lint)
777
778 uint64_t
779 get_jbus_config(void)
780 { return (0); }
781
782 /* ARGSUSED */
783 void
784 set_jbus_config(uint64_t jbus_config)
785 {}
786
787 /* ARGSUSED */
788 void
789 set_mcu_ctl_reg1(uint64_t mcu_ctl)
790 {}
791
792 uint64_t
793 get_mcu_ctl_reg1(void)
794 { return (0); }
795
796 #else /* lint */
797
798 ENTRY(get_jbus_config)
799 ldxa [%g0]ASI_JBUS_CONFIG, %o0
800 retl
801 nop
802 SET_SIZE(get_jbus_config)
803
804 ENTRY(set_jbus_config)
805 stxa %o0, [%g0]ASI_JBUS_CONFIG
806 membar #Sync
807 retl
808 nop
809 SET_SIZE(set_jbus_config)
810
811
812 ENTRY(get_mcu_ctl_reg1)
813 ldxa [%g0]ASI_MCU_CTRL, %o0 ! MCU control reg1 is at offset 0
814 retl
815 nop
816 SET_SIZE(get_mcu_ctl_reg1)
817
818
819 ENTRY(set_mcu_ctl_reg1)
820 stxa %o0, [%g0]ASI_MCU_CTRL ! MCU control reg1 is at offset 0
821 membar #Sync
822 retl
823 nop
824 SET_SIZE(set_mcu_ctl_reg1)
825
826 #endif /* lint */
827
828
829 #if defined(lint)
830 /*
831 * scrubphys - Pass in the aligned physical memory address
832 * that you want to scrub, along with the ecache set size.
833 *
834 * 1) Displacement flush the E$ line corresponding to %addr.
835 * The first ldxa guarantees that the %addr is no longer in
836 * M, O, or E (goes to I or S (if instruction fetch also happens).
837 * 2) "Write" the data using a CAS %addr,%g0,%g0.
838 * The casxa guarantees a transition from I to M or S to M.
839 * 3) Displacement flush the E$ line corresponding to %addr.
840 * The second ldxa pushes the M line out of the ecache, into the
841 * writeback buffers, on the way to memory.
842 * 4) The "membar #Sync" pushes the cache line out of the writeback
843 * buffers onto the bus, on the way to dram finally.
844 *
845 * This is a modified version of the algorithm suggested by Gary Lauterbach.
846 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
847 * as modified, but then we found out that for spitfire, if it misses in the
848 * E$ it will probably install as an M, but if it hits in the E$, then it
849 * will stay E, if the store doesn't happen. So the first displacement flush
850 * should ensure that the CAS will miss in the E$. Arrgh.
851 */
852 /* ARGSUSED */
853 void
854 scrubphys(uint64_t paddr, int ecache_set_size)
855 {}
856
857 #else /* lint */
858 ENTRY(scrubphys)
859 rdpr %pstate, %o4
860 andn %o4, PSTATE_IE | PSTATE_AM, %o5
861 wrpr %o5, %g0, %pstate ! clear IE, AM bits
862
863 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
864 CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_1);
865 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */
866 scrubphys_1:
867 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
868
869 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
870 casxa [%o0]ASI_MEM, %g0, %g0
871 ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
872
873 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
874 CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_2);
875 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */
876 scrubphys_2:
877 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
878
879 wrpr %g0, %o4, %pstate ! restore earlier pstate register value
880
881 retl
882 membar #Sync ! move the data out of the load buffer
883 SET_SIZE(scrubphys)
884
885 #endif /* lint */
886
887
888 #if defined(lint)
889 /*
890 * clearphys - Pass in the physical memory address of the checkblock
891 * that you want to push out, cleared with a recognizable pattern,
892 * from the ecache.
893 *
894 * To ensure that the ecc gets recalculated after the bad data is cleared,
895 * we must write out enough data to fill the w$ line (64 bytes). So we read
896 * in an entire ecache subblock's worth of data, and write it back out.
897 * Then we overwrite the 16 bytes of bad data with the pattern.
898 */
899 /* ARGSUSED */
900 void
901 clearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize)
902 {
903 }
904
905 #else /* lint */
906 ENTRY(clearphys)
907 /* turn off IE, AM bits */
908 rdpr %pstate, %o4
909 andn %o4, PSTATE_IE | PSTATE_AM, %o5
910 wrpr %o5, %g0, %pstate
911
912 /* turn off NCEEN */
913 ldxa [%g0]ASI_ESTATE_ERR, %o5
914 andn %o5, EN_REG_NCEEN, %o3
915 stxa %o3, [%g0]ASI_ESTATE_ERR
916 membar #Sync
917
918 /* align address passed with 64 bytes subblock size */
919 mov CH_ECACHE_SUBBLK_SIZE, %o2
920 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1
921
922 /* move the good data into the W$ */
923 1:
924 subcc %o2, 8, %o2
925 ldxa [%g1 + %o2]ASI_MEM, %g2
926 bge 1b
927 stxa %g2, [%g1 + %o2]ASI_MEM
928
929 /* now overwrite the bad data */
930 setx 0xbadecc00badecc01, %g1, %g2
931 stxa %g2, [%o0]ASI_MEM
932 mov 8, %g1
933 stxa %g2, [%o0 + %g1]ASI_MEM
934
935 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
936 CHK_JP_ERRATA85_ENABLED(%g1, clearphys_1);
937 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */
938 clearphys_1:
939 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
940
941 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
942 casxa [%o0]ASI_MEM, %g0, %g0
943 ECACHE_REFLUSH_LINE(%o1, %o2, %o3)
944
945 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
946 CHK_JP_ERRATA85_ENABLED(%g1, clearphys_2);
947 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */
948 clearphys_2:
949 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
950
951 /* clear the AFSR */
952 ldxa [%g0]ASI_AFSR, %o1
953 stxa %o1, [%g0]ASI_AFSR
954 membar #Sync
955
956 /* turn NCEEN back on */
957 stxa %o5, [%g0]ASI_ESTATE_ERR
958 membar #Sync
959
960 /* return and re-enable IE and AM */
961 retl
962 wrpr %g0, %o4, %pstate
963 SET_SIZE(clearphys)
964
965 #endif /* lint */
966
967
968 #if defined(lint)
969 /*
970 * Jalapeno Ecache displacement flush the specified line from the E$
971 *
972 * Register usage:
973 * %o0 - 64 bit physical address for flushing
974 * %o1 - Ecache set size
975 */
976 /*ARGSUSED*/
977 void
978 ecache_flush_line(uint64_t flushaddr, int ec_set_size)
979 {
980 }
981 #else /* lint */
982 ENTRY(ecache_flush_line)
983
984 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
985 CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_1);
986 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */
987 ecache_flush_line_1:
988 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
989
990 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
991
992 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
993 CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_2);
994 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */
995 ecache_flush_line_2:
996 #endif /* JALAPENO && JALAPENO_ERRATA_85 */
997
998 retl
999 nop
1000 SET_SIZE(ecache_flush_line)
1001 #endif /* lint */
1002
1003
1004 /*
1005 * Perform necessary cpu workaround to ensure jbus ordering.
1006 * Called only from Fire systems.
1007 * CPU's internal "invalidate FIFOs" are flushed.
1008 */
1009
1010 #if defined(lint)
1011 void
1012 jbus_stst_order()
1013 {}
1014 #else /* lint */
1015
1016 #define VIS_BLOCKSIZE 64
1017
1018 .seg ".data"
1019 .align VIS_BLOCKSIZE
1020 .type sync_buf, #object
1021 sync_buf:
1022 .skip VIS_BLOCKSIZE
1023 .size sync_buf, VIS_BLOCKSIZE
1024
1025 ENTRY(jbus_stst_order)
1026 set sync_buf, %o1
1027
1028 rd %fprs, %o2 ! %o2 = saved fprs
1029 or %o2, FPRS_FEF, %o3
1030 wr %g0, %o3, %fprs ! make sure fp is enabled
1031 stda %d0, [%o1]ASI_BLK_COMMIT_P
1032 wr %o2, 0, %fprs ! restore fprs
1033
1034 retl
1035 membar #Sync
1036 SET_SIZE(jbus_stst_order)
1037
1038 #endif /* lint */
1039
1040 #if defined(lint)
1041 /*
1042 * This routine will not be called in Jalapeno systems.
1043 */
1044 void
1045 flush_ipb(void)
1046 { return; }
1047
1048 #else /* lint */
1049
1050 ENTRY(flush_ipb)
1051 retl
1052 nop
1053 SET_SIZE(flush_ipb)
1054
1055 #endif /* lint */