8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Assembly code support for the Cheetah+ module
26 */
27
28 #pragma ident "%Z%%M% %I% %E% SMI"
29
30 #if !defined(lint)
31 #include "assym.h"
32 #endif /* lint */
33
34 #include <sys/asm_linkage.h>
35 #include <sys/mmu.h>
36 #include <vm/hat_sfmmu.h>
37 #include <sys/machparam.h>
38 #include <sys/machcpuvar.h>
39 #include <sys/machthread.h>
40 #include <sys/machtrap.h>
41 #include <sys/privregs.h>
42 #include <sys/asm_linkage.h>
43 #include <sys/trap.h>
44 #include <sys/cheetahregs.h>
45 #include <sys/xc_impl.h>
46 #include <sys/intreg.h>
47 #include <sys/async.h>
48 #include <sys/clock.h>
49 #include <sys/cheetahasm.h>
50 #include <sys/cmpregs.h>
51
52 #ifdef TRAPTRACE
53 #include <sys/traptrace.h>
54 #endif /* TRAPTRACE */
55
56
57 #if !defined(lint)
58
59 .global retire_l2_start
60 .global retire_l2_end
61 .global unretire_l2_start
62 .global unretire_l2_end
63 .global retire_l3_start
64 .global retire_l3_end
65 .global unretire_l3_start
66 .global unretire_l3_end
67
68 /*
69 * Panther version to reflush a line from both the L2 cache and L3
70 * cache by the respective indexes. Flushes all ways of the line from
71 * each cache.
72 *
73 * l2_index Index into the L2$ of the line to be flushed. This
74 * register will not be modified by this routine.
75 * l3_index Index into the L3$ of the line to be flushed. This
76 * register will not be modified by this routine.
77 * scr2 scratch register.
78 * scr3 scratch register.
110 * l3_idx_out Input: scratch register.
111 * Output: Index into the L3$ of the line to be flushed.
112 * scr3 scratch register.
113 * scr4 scratch register.
114 *
115 */
116 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \
117 set PN_L3_SET_SIZE, l2_idx_out; \
118 sub l2_idx_out, 1, l2_idx_out; \
119 and physaddr, l2_idx_out, l3_idx_out; \
120 set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \
121 or l2_idx_out, l3_idx_out, l3_idx_out; \
122 set PN_L2_SET_SIZE, l2_idx_out; \
123 sub l2_idx_out, 1, l2_idx_out; \
124 and physaddr, l2_idx_out, l2_idx_out; \
125 set PN_L2_IDX_DISP_FLUSH, scr3; \
126 or l2_idx_out, scr3, l2_idx_out; \
127 PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
128
129
130 #endif /* !lint */
131
132 #if defined(lint)
133
134 /*ARGSUSED*/
135 int
136 retire_l2(uint64_t tag_addr, uint64_t pattern)
137 {return 0;}
138
139 #else
140 .align 4096
141 ENTRY(retire_l2)
142 retire_l2_start:
143
144 ! since we disable interrupts, we don't need to do kpreempt_disable()
145 rdpr %pstate, %o2
146 andn %o2, PSTATE_IE, %g1
147 wrpr %g0, %g1, %pstate ! disable interrupts
148 /*
149 * Save current DCU state. Turn off IPS
150 */
151 setx DCU_IPS_MASK, %g2, %o3
152 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
153 andn %g1, %o3, %g4
154 stxa %g4, [%g0]ASI_DCU
155 flush %g0
156 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
157 clr %o5 ! assume success
158 8:
159 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3)
181 wrpr %g0, %o2, %pstate !restore pstate
182 retl
183 mov %o5, %o0
184 2:
185 ! It is OK to have STATE as NA (if so, nothing to do!)
186 and %o3, 0x7, %o3
187 cmp %o3, 0x5
188 be,a,pt %xcc, 9b
189 mov 1, %o5 ! indicate was already NA
190 ! Hmm. Not INV, not NA.
191 cmp %o5, 0
192 be,a,pt %xcc, 8b ! Flush the cacheline again
193 mov 2, %o5 ! indicate retry was done
194 ! We already Flushed cacheline second time. Return -1
195 clr %o5
196 ba 9b
197 dec %o5
198 retire_l2_end:
199 SET_SIZE(retire_l2)
200
201 #endif /* lint */
202
203 #if defined(lint)
204
205 /*
206 */
207 /*ARGSUSED*/
208 int
209 unretire_l2(uint64_t tag_addr, uint64_t pattern)
210 {return 0;}
211
212 #else
213 ENTRY(unretire_l2)
214 unretire_l2_start:
215
216 ! since we disable interrupts, we don't need to do kpreempt_disable()
217 rdpr %pstate, %o2
218 andn %o2, PSTATE_IE, %g1
219 wrpr %g0, %g1, %pstate ! disable interrupts
220 /*
221 * Save current DCU state. Turn off IPS
222 */
223 setx DCU_IPS_MASK, %g2, %o3
224 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
225 andn %g1, %o3, %g4
226 stxa %g4, [%g0]ASI_DCU
227 flush %g0 /* flush required after changing the IC bit */
228 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
229
230 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
231 1:
232 clr %o5 ! assume success
240 membar #Sync
241 ! now delay 15 cycles so we don't have hazard when we return
242 mov 16, %o1
243 1:
244 brnz,pt %o1, 1b
245 dec %o1
246 9:
247 ! UNPARK-SIBLING_CORE is 7 instructions
248 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
249 /*
250 * Restore the DCU
251 */
252 stxa %g1, [%g0]ASI_DCU
253 flush %g0
254 wrpr %g0, %o2, %pstate !restore pstate
255 retl
256 mov %o5, %o0
257 unretire_l2_end:
258 SET_SIZE(unretire_l2)
259
260 #endif /* lint */
261
262 #if defined(lint)
263
264 /*ARGSUSED*/
265 int
266 retire_l3(uint64_t tag_addr, uint64_t pattern)
267 {return 0;}
268
269 #else
270 ENTRY(retire_l3)
271 retire_l3_start:
272
273 ! since we disable interrupts, we don't need to do kpreempt_disable()
274 rdpr %pstate, %o2
275 andn %o2, PSTATE_IE, %g1
276 wrpr %g0, %g1, %pstate ! disable interrupts
277 /*
278 * Save current DCU state. Turn off IPS
279 */
280 setx DCU_IPS_MASK, %g2, %o3
281 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
282 andn %g1, %o3, %g4
283 stxa %g4, [%g0]ASI_DCU
284 flush %g0 /* flush required after changing the IC bit */
285 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
286
287 ! PN-ECACHE-FLUSH_LINE is 30 instructions
288 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
289 1:
306 /*
307 * Restore the DCU
308 */
309 stxa %g1, [%g0]ASI_DCU
310 flush %g0
311 wrpr %g0, %o2, %pstate !restore pstate
312 retl
313 mov %o5, %o0
314 2:
315 ! It is OK to have STATE as NA (if so, nothing to do!)
316 and %o3, 0x7, %o3
317 cmp %o3, 0x5
318 be,a,pt %xcc, 9b
319 inc %o5 ! indicate was already NA
320 ! Hmm. Not INV, not NA
321 ba 9b
322 dec %o5
323 retire_l3_end:
324 SET_SIZE(retire_l3)
325
326 #endif /* lint */
327
328 #if defined(lint)
329
330 /*
331 */
332 /*ARGSUSED*/
333 int
334 unretire_l3(uint64_t tag_addr, uint64_t pattern)
335 {return 0;}
336
337 #else
338 ENTRY(unretire_l3)
339 unretire_l3_start:
340
341 ! since we disable interrupts, we don't need to do kpreempt_disable()
342 rdpr %pstate, %o2
343 andn %o2, PSTATE_IE, %g1
344 wrpr %g0, %g1, %pstate ! disable interrupts
345 /*
346 * Save current DCU state. Turn off IPS
347 */
348 setx DCU_IPS_MASK, %g2, %o3
349 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
350 andn %g1, %o3, %g4
351 stxa %g4, [%g0]ASI_DCU
352 flush %g0 /* flush required after changing the IC bit */
353 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
354
355 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
356 1:
357 clr %o5 ! assume success
365 membar #Sync
366 ! now delay 15 cycles so we don't have hazard when we return
367 mov 16, %o1
368 1:
369 brnz,pt %o1, 1b
370 dec %o1
371 9:
372 ! UNPARK-SIBLING_CORE is 7 instructions
373 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
374 /*
375 * Restore the DCU
376 */
377 stxa %g1, [%g0]ASI_DCU
378 flush %g0
379 wrpr %g0, %o2, %pstate !restore pstate
380 retl
381 mov %o5, %o0
382 unretire_l3_end:
383 SET_SIZE(unretire_l3)
384
385 #endif /* lint */
386
387 #if defined(lint)
388
389 /*ARGSUSED*/
390 int
391 retire_l2_alternate(uint64_t tag_addr, uint64_t pattern)
392 {return 0;}
393
394 #else
395 .align 2048
396
397 ENTRY(retire_l2_alternate)
398
399 ! since we disable interrupts, we don't need to do kpreempt_disable()
400 rdpr %pstate, %o2
401 andn %o2, PSTATE_IE, %g1
402 wrpr %g0, %g1, %pstate ! disable interrupts
403 /*
404 * Save current DCU state. Turn off IPS
405 */
406 setx DCU_IPS_MASK, %g2, %o3
407 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
408 andn %g1, %o3, %g4
409 stxa %g4, [%g0]ASI_DCU
410 flush %g0
411 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
412 clr %o5 ! assume success
413 8:
414 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3)
435 flush %g0
436 wrpr %g0, %o2, %pstate !restore pstate
437 retl
438 mov %o5, %o0
439 2:
440 ! It is OK to have STATE as NA (if so, nothing to do!)
441 and %o3, 0x7, %o3
442 cmp %o3, 0x5
443 be,a,pt %xcc, 9b
444 mov 1, %o5 ! indicate was already NA
445 ! Hmm. Not INV, not NA.
446 cmp %o5, 0
447 be,a,pt %xcc, 8b ! Flush the cacheline again
448 mov 2, %o5 ! indicate retry was done
449 ! We already Flushed cacheline second time. Return -1
450 clr %o5
451 ba 9b
452 dec %o5
453 SET_SIZE(retire_l2_alternate)
454
455 #endif /* lint */
456
457 #if defined(lint)
458
459 /*
460 */
461 /*ARGSUSED*/
462 int
463 unretire_l2_alternate(uint64_t tag_addr, uint64_t pattern)
464 {return 0;}
465
466 #else
467 ENTRY(unretire_l2_alternate)
468
469 ! since we disable interrupts, we don't need to do kpreempt_disable()
470 rdpr %pstate, %o2
471 andn %o2, PSTATE_IE, %g1
472 wrpr %g0, %g1, %pstate ! disable interrupts
473 /*
474 * Save current DCU state. Turn off IPS
475 */
476 setx DCU_IPS_MASK, %g2, %o3
477 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
478 andn %g1, %o3, %g4
479 stxa %g4, [%g0]ASI_DCU
480 flush %g0 /* flush required after changing the IC bit */
481 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
482
483 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
484 1:
485 clr %o5 ! assume success
486 ! Check that line is in NA state; if so, INV it.
492 stxa %g0, [%o0]ASI_L2_TAG
493 membar #Sync
494 ! now delay 15 cycles so we don't have hazard when we return
495 mov 16, %o1
496 1:
497 brnz,pt %o1, 1b
498 dec %o1
499 9:
500 ! UNPARK-SIBLING_CORE is 7 instructions
501 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
502 /*
503 * Restore the DCU
504 */
505 stxa %g1, [%g0]ASI_DCU
506 flush %g0
507 wrpr %g0, %o2, %pstate !restore pstate
508 retl
509 mov %o5, %o0
510 SET_SIZE(unretire_l2_alternate)
511
512 #endif /* lint */
513
514 #if defined(lint)
515
516 /*ARGSUSED*/
517 int
518 retire_l3_alternate(uint64_t tag_addr, uint64_t pattern)
519 {return 0;}
520
521 #else
522 ENTRY(retire_l3_alternate)
523
524 ! since we disable interrupts, we don't need to do kpreempt_disable()
525 rdpr %pstate, %o2
526 andn %o2, PSTATE_IE, %g1
527 wrpr %g0, %g1, %pstate ! disable interrupts
528 /*
529 * Save current DCU state. Turn off IPS
530 */
531 setx DCU_IPS_MASK, %g2, %o3
532 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
533 andn %g1, %o3, %g4
534 stxa %g4, [%g0]ASI_DCU
535 flush %g0 /* flush required after changing the IC bit */
536 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
537
538 ! PN-ECACHE-FLUSH_LINE is 30 instructions
539 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
540 1:
541 clr %o5 ! assume success
556 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
557 /*
558 * Restore the DCU
559 */
560 stxa %g1, [%g0]ASI_DCU
561 flush %g0
562 wrpr %g0, %o2, %pstate !restore pstate
563 retl
564 mov %o5, %o0
565 2:
566 ! It is OK to have STATE as NA (if so, nothing to do!)
567 and %o3, 0x7, %o3
568 cmp %o3, 0x5
569 be,a,pt %xcc, 9b
570 inc %o5 ! indicate was already NA
571 ! Hmm. Not INV, not NA
572 ba 9b
573 dec %o5
574 SET_SIZE(retire_l3_alternate)
575
576 #endif /* lint */
577
578 #if defined(lint)
579
580 /*
581 */
582 /*ARGSUSED*/
583 int
584 unretire_l3_alternate(uint64_t tag_addr, uint64_t pattern)
585 {return 0;}
586
587 #else
588 ENTRY(unretire_l3_alternate)
589
590 ! since we disable interrupts, we don't need to do kpreempt_disable()
591 rdpr %pstate, %o2
592 andn %o2, PSTATE_IE, %g1
593 wrpr %g0, %g1, %pstate ! disable interrupts
594 /*
595 * Save current DCU state. Turn off IPS
596 */
597 setx DCU_IPS_MASK, %g2, %o3
598 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
599 andn %g1, %o3, %g4
600 stxa %g4, [%g0]ASI_DCU
601 flush %g0 /* flush required after changing the IC bit */
602 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
603
604 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
605 1:
606 clr %o5 ! assume success
607 ! Check that line is in NA state; if so, INV it.
613 stxa %g0, [%o0]ASI_EC_DIAG
614 membar #Sync
615 ! now delay 15 cycles so we don't have hazard when we return
616 mov 16, %o1
617 1:
618 brnz,pt %o1, 1b
619 dec %o1
620 9:
621 ! UNPARK-SIBLING_CORE is 7 instructions
622 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
623 /*
624 * Restore the DCU
625 */
626 stxa %g1, [%g0]ASI_DCU
627 flush %g0
628 wrpr %g0, %o2, %pstate !restore pstate
629 retl
630 mov %o5, %o0
631 SET_SIZE(unretire_l3_alternate)
632
633 #endif /* lint */
634
635 #if defined(lint)
636
637 /*ARGSUSED*/
638 void
639 get_ecache_dtags_tl1(uint64_t afar, ch_cpu_logout_t *clop)
640 { }
641
642 #else
643 ENTRY(get_ecache_dtags_tl1)
644
645
646 PARK_SIBLING_CORE(%g3, %g4, %g5)
647 add %g2, CH_CLO_DATA + CH_CHD_EC_DATA, %g2
648 rd %asi, %g4
649 wr %g0, ASI_N, %asi
650 GET_ECACHE_DTAGS(%g1, %g2, %g5, %g6, %g7)
651 wr %g4, %asi
652 UNPARK_SIBLING_CORE(%g3, %g4, %g5) ! can use %g3 again
653
654 retry
655 SET_SIZE(get_ecache_dtags_tl1)
656
657 #endif /* lint */
658
659 #if defined(lint)
660 /*ARGSUSED*/
661 void
662 get_l2_tag_tl1(uint64_t tag_addr, uint64_t tag_data_ptr)
663 { }
664
665 #else
666 ENTRY(get_l2_tag_tl1)
667
668 /*
669 * Now read the tag data
670 */
671 ldxa [%g1]ASI_L2_TAG, %g4 ! save tag_data
672 stx %g4, [%g2]
673
674 retry
675 SET_SIZE(get_l2_tag_tl1)
676
677 #endif /* lint */
678
679 #if defined(lint)
680 /*ARGSUSED*/
681 void
682 get_l3_tag_tl1(uint64_t tag_addr, uint64_t tag_data_ptr)
683 { }
684
685 #else
686 ENTRY(get_l3_tag_tl1)
687
688 /*
689 * Now read the tag data
690 */
691 ldxa [%g1]ASI_EC_DIAG, %g4 ! save tag_data
692 stx %g4, [%g2]
693
694 retry
695 SET_SIZE(get_l3_tag_tl1)
696
697 #endif /* lint */
698
|
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Assembly code support for the Cheetah+ module
26 */
27
28 #include "assym.h"
29
30 #include <sys/asm_linkage.h>
31 #include <sys/mmu.h>
32 #include <vm/hat_sfmmu.h>
33 #include <sys/machparam.h>
34 #include <sys/machcpuvar.h>
35 #include <sys/machthread.h>
36 #include <sys/machtrap.h>
37 #include <sys/privregs.h>
38 #include <sys/asm_linkage.h>
39 #include <sys/trap.h>
40 #include <sys/cheetahregs.h>
41 #include <sys/xc_impl.h>
42 #include <sys/intreg.h>
43 #include <sys/async.h>
44 #include <sys/clock.h>
45 #include <sys/cheetahasm.h>
46 #include <sys/cmpregs.h>
47
48 #ifdef TRAPTRACE
49 #include <sys/traptrace.h>
50 #endif /* TRAPTRACE */
51
52
53 .global retire_l2_start
54 .global retire_l2_end
55 .global unretire_l2_start
56 .global unretire_l2_end
57 .global retire_l3_start
58 .global retire_l3_end
59 .global unretire_l3_start
60 .global unretire_l3_end
61
62 /*
63 * Panther version to reflush a line from both the L2 cache and L3
64 * cache by the respective indexes. Flushes all ways of the line from
65 * each cache.
66 *
67 * l2_index Index into the L2$ of the line to be flushed. This
68 * register will not be modified by this routine.
69 * l3_index Index into the L3$ of the line to be flushed. This
70 * register will not be modified by this routine.
71 * scr2 scratch register.
72 * scr3 scratch register.
104 * l3_idx_out Input: scratch register.
105 * Output: Index into the L3$ of the line to be flushed.
106 * scr3 scratch register.
107 * scr4 scratch register.
108 *
109 */
110 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \
111 set PN_L3_SET_SIZE, l2_idx_out; \
112 sub l2_idx_out, 1, l2_idx_out; \
113 and physaddr, l2_idx_out, l3_idx_out; \
114 set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \
115 or l2_idx_out, l3_idx_out, l3_idx_out; \
116 set PN_L2_SET_SIZE, l2_idx_out; \
117 sub l2_idx_out, 1, l2_idx_out; \
118 and physaddr, l2_idx_out, l2_idx_out; \
119 set PN_L2_IDX_DISP_FLUSH, scr3; \
120 or l2_idx_out, scr3, l2_idx_out; \
121 PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
122
123
124 .align 4096
125 ENTRY(retire_l2)
126 retire_l2_start:
127
128 ! since we disable interrupts, we don't need to do kpreempt_disable()
129 rdpr %pstate, %o2
130 andn %o2, PSTATE_IE, %g1
131 wrpr %g0, %g1, %pstate ! disable interrupts
132 /*
133 * Save current DCU state. Turn off IPS
134 */
135 setx DCU_IPS_MASK, %g2, %o3
136 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
137 andn %g1, %o3, %g4
138 stxa %g4, [%g0]ASI_DCU
139 flush %g0
140 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
141 clr %o5 ! assume success
142 8:
143 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3)
165 wrpr %g0, %o2, %pstate !restore pstate
166 retl
167 mov %o5, %o0
168 2:
169 ! It is OK to have STATE as NA (if so, nothing to do!)
170 and %o3, 0x7, %o3
171 cmp %o3, 0x5
172 be,a,pt %xcc, 9b
173 mov 1, %o5 ! indicate was already NA
174 ! Hmm. Not INV, not NA.
175 cmp %o5, 0
176 be,a,pt %xcc, 8b ! Flush the cacheline again
177 mov 2, %o5 ! indicate retry was done
178 ! We already Flushed cacheline second time. Return -1
179 clr %o5
180 ba 9b
181 dec %o5
182 retire_l2_end:
183 SET_SIZE(retire_l2)
184
185 ENTRY(unretire_l2)
186 unretire_l2_start:
187
188 ! since we disable interrupts, we don't need to do kpreempt_disable()
189 rdpr %pstate, %o2
190 andn %o2, PSTATE_IE, %g1
191 wrpr %g0, %g1, %pstate ! disable interrupts
192 /*
193 * Save current DCU state. Turn off IPS
194 */
195 setx DCU_IPS_MASK, %g2, %o3
196 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
197 andn %g1, %o3, %g4
198 stxa %g4, [%g0]ASI_DCU
199 flush %g0 /* flush required after changing the IC bit */
200 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
201
202 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
203 1:
204 clr %o5 ! assume success
212 membar #Sync
213 ! now delay 15 cycles so we don't have hazard when we return
214 mov 16, %o1
215 1:
216 brnz,pt %o1, 1b
217 dec %o1
218 9:
219 ! UNPARK-SIBLING_CORE is 7 instructions
220 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
221 /*
222 * Restore the DCU
223 */
224 stxa %g1, [%g0]ASI_DCU
225 flush %g0
226 wrpr %g0, %o2, %pstate !restore pstate
227 retl
228 mov %o5, %o0
229 unretire_l2_end:
230 SET_SIZE(unretire_l2)
231
232 ENTRY(retire_l3)
233 retire_l3_start:
234
235 ! since we disable interrupts, we don't need to do kpreempt_disable()
236 rdpr %pstate, %o2
237 andn %o2, PSTATE_IE, %g1
238 wrpr %g0, %g1, %pstate ! disable interrupts
239 /*
240 * Save current DCU state. Turn off IPS
241 */
242 setx DCU_IPS_MASK, %g2, %o3
243 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
244 andn %g1, %o3, %g4
245 stxa %g4, [%g0]ASI_DCU
246 flush %g0 /* flush required after changing the IC bit */
247 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
248
249 ! PN-ECACHE-FLUSH_LINE is 30 instructions
250 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
251 1:
268 /*
269 * Restore the DCU
270 */
271 stxa %g1, [%g0]ASI_DCU
272 flush %g0
273 wrpr %g0, %o2, %pstate !restore pstate
274 retl
275 mov %o5, %o0
276 2:
277 ! It is OK to have STATE as NA (if so, nothing to do!)
278 and %o3, 0x7, %o3
279 cmp %o3, 0x5
280 be,a,pt %xcc, 9b
281 inc %o5 ! indicate was already NA
282 ! Hmm. Not INV, not NA
283 ba 9b
284 dec %o5
285 retire_l3_end:
286 SET_SIZE(retire_l3)
287
288 ENTRY(unretire_l3)
289 unretire_l3_start:
290
291 ! since we disable interrupts, we don't need to do kpreempt_disable()
292 rdpr %pstate, %o2
293 andn %o2, PSTATE_IE, %g1
294 wrpr %g0, %g1, %pstate ! disable interrupts
295 /*
296 * Save current DCU state. Turn off IPS
297 */
298 setx DCU_IPS_MASK, %g2, %o3
299 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
300 andn %g1, %o3, %g4
301 stxa %g4, [%g0]ASI_DCU
302 flush %g0 /* flush required after changing the IC bit */
303 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
304
305 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
306 1:
307 clr %o5 ! assume success
315 membar #Sync
316 ! now delay 15 cycles so we don't have hazard when we return
317 mov 16, %o1
318 1:
319 brnz,pt %o1, 1b
320 dec %o1
321 9:
322 ! UNPARK-SIBLING_CORE is 7 instructions
323 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
324 /*
325 * Restore the DCU
326 */
327 stxa %g1, [%g0]ASI_DCU
328 flush %g0
329 wrpr %g0, %o2, %pstate !restore pstate
330 retl
331 mov %o5, %o0
332 unretire_l3_end:
333 SET_SIZE(unretire_l3)
334
335 .align 2048
336
337 ENTRY(retire_l2_alternate)
338
339 ! since we disable interrupts, we don't need to do kpreempt_disable()
340 rdpr %pstate, %o2
341 andn %o2, PSTATE_IE, %g1
342 wrpr %g0, %g1, %pstate ! disable interrupts
343 /*
344 * Save current DCU state. Turn off IPS
345 */
346 setx DCU_IPS_MASK, %g2, %o3
347 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
348 andn %g1, %o3, %g4
349 stxa %g4, [%g0]ASI_DCU
350 flush %g0
351 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
352 clr %o5 ! assume success
353 8:
354 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3)
375 flush %g0
376 wrpr %g0, %o2, %pstate !restore pstate
377 retl
378 mov %o5, %o0
379 2:
380 ! It is OK to have STATE as NA (if so, nothing to do!)
381 and %o3, 0x7, %o3
382 cmp %o3, 0x5
383 be,a,pt %xcc, 9b
384 mov 1, %o5 ! indicate was already NA
385 ! Hmm. Not INV, not NA.
386 cmp %o5, 0
387 be,a,pt %xcc, 8b ! Flush the cacheline again
388 mov 2, %o5 ! indicate retry was done
389 ! We already Flushed cacheline second time. Return -1
390 clr %o5
391 ba 9b
392 dec %o5
393 SET_SIZE(retire_l2_alternate)
394
395 ENTRY(unretire_l2_alternate)
396
397 ! since we disable interrupts, we don't need to do kpreempt_disable()
398 rdpr %pstate, %o2
399 andn %o2, PSTATE_IE, %g1
400 wrpr %g0, %g1, %pstate ! disable interrupts
401 /*
402 * Save current DCU state. Turn off IPS
403 */
404 setx DCU_IPS_MASK, %g2, %o3
405 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
406 andn %g1, %o3, %g4
407 stxa %g4, [%g0]ASI_DCU
408 flush %g0 /* flush required after changing the IC bit */
409 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
410
411 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
412 1:
413 clr %o5 ! assume success
414 ! Check that line is in NA state; if so, INV it.
420 stxa %g0, [%o0]ASI_L2_TAG
421 membar #Sync
422 ! now delay 15 cycles so we don't have hazard when we return
423 mov 16, %o1
424 1:
425 brnz,pt %o1, 1b
426 dec %o1
427 9:
428 ! UNPARK-SIBLING_CORE is 7 instructions
429 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
430 /*
431 * Restore the DCU
432 */
433 stxa %g1, [%g0]ASI_DCU
434 flush %g0
435 wrpr %g0, %o2, %pstate !restore pstate
436 retl
437 mov %o5, %o0
438 SET_SIZE(unretire_l2_alternate)
439
440 ENTRY(retire_l3_alternate)
441
442 ! since we disable interrupts, we don't need to do kpreempt_disable()
443 rdpr %pstate, %o2
444 andn %o2, PSTATE_IE, %g1
445 wrpr %g0, %g1, %pstate ! disable interrupts
446 /*
447 * Save current DCU state. Turn off IPS
448 */
449 setx DCU_IPS_MASK, %g2, %o3
450 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
451 andn %g1, %o3, %g4
452 stxa %g4, [%g0]ASI_DCU
453 flush %g0 /* flush required after changing the IC bit */
454 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
455
456 ! PN-ECACHE-FLUSH_LINE is 30 instructions
457 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
458 1:
459 clr %o5 ! assume success
474 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
475 /*
476 * Restore the DCU
477 */
478 stxa %g1, [%g0]ASI_DCU
479 flush %g0
480 wrpr %g0, %o2, %pstate !restore pstate
481 retl
482 mov %o5, %o0
483 2:
484 ! It is OK to have STATE as NA (if so, nothing to do!)
485 and %o3, 0x7, %o3
486 cmp %o3, 0x5
487 be,a,pt %xcc, 9b
488 inc %o5 ! indicate was already NA
489 ! Hmm. Not INV, not NA
490 ba 9b
491 dec %o5
492 SET_SIZE(retire_l3_alternate)
493
494 ENTRY(unretire_l3_alternate)
495
496 ! since we disable interrupts, we don't need to do kpreempt_disable()
497 rdpr %pstate, %o2
498 andn %o2, PSTATE_IE, %g1
499 wrpr %g0, %g1, %pstate ! disable interrupts
500 /*
501 * Save current DCU state. Turn off IPS
502 */
503 setx DCU_IPS_MASK, %g2, %o3
504 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1
505 andn %g1, %o3, %g4
506 stxa %g4, [%g0]ASI_DCU
507 flush %g0 /* flush required after changing the IC bit */
508 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value
509
510 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
511 1:
512 clr %o5 ! assume success
513 ! Check that line is in NA state; if so, INV it.
519 stxa %g0, [%o0]ASI_EC_DIAG
520 membar #Sync
521 ! now delay 15 cycles so we don't have hazard when we return
522 mov 16, %o1
523 1:
524 brnz,pt %o1, 1b
525 dec %o1
526 9:
527 ! UNPARK-SIBLING_CORE is 7 instructions
528 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions
529 /*
530 * Restore the DCU
531 */
532 stxa %g1, [%g0]ASI_DCU
533 flush %g0
534 wrpr %g0, %o2, %pstate !restore pstate
535 retl
536 mov %o5, %o0
537 SET_SIZE(unretire_l3_alternate)
538
539 ENTRY(get_ecache_dtags_tl1)
540
541
542 PARK_SIBLING_CORE(%g3, %g4, %g5)
543 add %g2, CH_CLO_DATA + CH_CHD_EC_DATA, %g2
544 rd %asi, %g4
545 wr %g0, ASI_N, %asi
546 GET_ECACHE_DTAGS(%g1, %g2, %g5, %g6, %g7)
547 wr %g4, %asi
548 UNPARK_SIBLING_CORE(%g3, %g4, %g5) ! can use %g3 again
549
550 retry
551 SET_SIZE(get_ecache_dtags_tl1)
552
553 ENTRY(get_l2_tag_tl1)
554
555 /*
556 * Now read the tag data
557 */
558 ldxa [%g1]ASI_L2_TAG, %g4 ! save tag_data
559 stx %g4, [%g2]
560
561 retry
562 SET_SIZE(get_l2_tag_tl1)
563
564 ENTRY(get_l3_tag_tl1)
565
566 /*
567 * Now read the tag data
568 */
569 ldxa [%g1]ASI_EC_DIAG, %g4 ! save tag_data
570 stx %g4, [%g2]
571
572 retry
573 SET_SIZE(get_l3_tag_tl1)
574
|