1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * SFMMU primitives. These primitives should only be used by sfmmu
28 * routines.
29 */
30
31 #include "assym.h"
32
33 #include <sys/asm_linkage.h>
34 #include <sys/machtrap.h>
35 #include <sys/machasi.h>
36 #include <sys/sun4asi.h>
37 #include <sys/pte.h>
38 #include <sys/mmu.h>
39 #include <vm/hat_sfmmu.h>
40 #include <vm/seg_spt.h>
41 #include <sys/machparam.h>
42 #include <sys/privregs.h>
43 #include <sys/scb.h>
44 #include <sys/intreg.h>
45 #include <sys/machthread.h>
46 #include <sys/clock.h>
47 #include <sys/trapstat.h>
48
49 /*
50 * sfmmu related subroutines
51 */
52
53 /*
54 * Invalidate either the context of a specific victim or any process
55 * currently running on this CPU.
56 *
57 * %g1 = sfmmup whose ctx is being invalidated
58 * when called from sfmmu_wrap_around, %g1 == INVALID_CONTEXT
59 * Note %g1 is the only input argument used by this xcall handler.
60 */
61 ENTRY(sfmmu_raise_tsb_exception)
62 !
63 ! if (victim == INVALID_CONTEXT ||
64 ! current CPU tsbmiss->usfmmup == victim sfmmup) {
65 ! if (shctx_on) {
66 ! shctx = INVALID;
67 ! }
68 ! if (sec-ctx > INVALID_CONTEXT) {
69 ! write INVALID_CONTEXT to sec-ctx
70 ! }
71 ! if (pri-ctx > INVALID_CONTEXT) {
72 ! write INVALID_CONTEXT to pri-ctx
73 ! }
74 ! }
75
76 sethi %hi(ksfmmup), %g3
77 ldx [%g3 + %lo(ksfmmup)], %g3
78 cmp %g1, %g3
79 be,a,pn %xcc, ptl1_panic /* can't invalidate kernel ctx */
80 mov PTL1_BAD_RAISE_TSBEXCP, %g1
81
82 set INVALID_CONTEXT, %g2
83 cmp %g1, INVALID_CONTEXT
84 be,pn %xcc, 0f /* called from wrap_around? */
85 mov MMU_SCONTEXT, %g3
86
87 CPU_TSBMISS_AREA(%g5, %g6) /* load cpu tsbmiss area */
88 ldx [%g5 + TSBMISS_UHATID], %g5 /* load usfmmup */
89 cmp %g5, %g1 /* hat toBe-invalid running? */
90 bne,pt %xcc, 3f
91 nop
92
93 0:
94 sethi %hi(shctx_on), %g5
95 ld [%g5 + %lo(shctx_on)], %g5
96 brz %g5, 1f
97 mov MMU_SHARED_CONTEXT, %g5
98 sethi %hi(FLUSH_ADDR), %g4
99 stxa %g0, [%g5]ASI_MMU_CTX
100 flush %g4
101
102 1:
103 ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = pgsz | sec-ctx */
104 set CTXREG_CTX_MASK, %g4
105 and %g5, %g4, %g5 /* %g5 = sec-ctx */
106 cmp %g5, INVALID_CONTEXT /* kernel ctx or invald ctx? */
107 ble,pn %xcc, 2f /* yes, no need to change */
108 mov MMU_PCONTEXT, %g7
109
110 stxa %g2, [%g3]ASI_MMU_CTX /* set invalid ctx */
111 membar #Sync
112
113 2:
114 ldxa [%g7]ASI_MMU_CTX, %g3 /* get pgz | pri-ctx */
115 and %g3, %g4, %g5 /* %g5 = pri-ctx */
116 cmp %g5, INVALID_CONTEXT /* kernel ctx or invald ctx? */
117 ble,pn %xcc, 3f /* yes, no need to change */
118 srlx %g3, CTXREG_NEXT_SHIFT, %g3 /* %g3 = nucleus pgsz */
119 sllx %g3, CTXREG_NEXT_SHIFT, %g3 /* need to preserve nucleus pgsz */
120 or %g3, %g2, %g2 /* %g2 = nucleus pgsz | INVALID_CONTEXT */
121
122 stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */
123 3:
124 retry
125 SET_SIZE(sfmmu_raise_tsb_exception)
126
127
128
129 /*
130 * %o0 = virtual address
131 * %o1 = address of TTE to be loaded
132 */
133 ENTRY_NP(sfmmu_itlb_ld_kva)
134 rdpr %pstate, %o3
135 #ifdef DEBUG
136 PANIC_IF_INTR_DISABLED_PSTR(%o3, msfmmu_di_l1, %g1)
137 #endif /* DEBUG */
138 wrpr %o3, PSTATE_IE, %pstate ! Disable interrupts
139 srln %o0, MMU_PAGESHIFT, %o0
140 slln %o0, MMU_PAGESHIFT, %o0 ! Clear page offset
141
142 ldx [%o1], %g1
143 set MMU_TAG_ACCESS, %o5
144 #ifdef CHEETAHPLUS_ERRATUM_34
145 !
146 ! If this is Cheetah or derivative and the specified TTE is locked
147 ! and hence to be loaded into the T16, fully-associative TLB, we
148 ! must avoid Cheetah+ erratum 34. In Cheetah+ erratum 34, under
149 ! certain conditions an ITLB locked index 0 TTE will erroneously be
150 ! displaced when a new TTE is loaded via ASI_ITLB_IN. To avoid
151 ! this erratum, we scan the T16 top down for an unlocked TTE and
152 ! explicitly load the specified TTE into that index.
153 !
154 GET_CPU_IMPL(%g2)
155 cmp %g2, CHEETAH_IMPL
156 bl,pn %icc, 0f
157 nop
158
159 andcc %g1, TTE_LCK_INT, %g0
160 bz %icc, 0f ! Lock bit is not set;
161 ! load normally.
162 or %g0, (15 << 3), %g3 ! Start searching from the
163 ! top down.
164
165 1:
166 ldxa [%g3]ASI_ITLB_ACCESS, %g4 ! Load TTE from t16
167
168 !
169 ! If this entry isn't valid, we'll choose to displace it (regardless
170 ! of the lock bit).
171 !
172 cmp %g4, %g0
173 bge %xcc, 2f ! TTE is > 0 iff not valid
174 andcc %g4, TTE_LCK_INT, %g0 ! Check for lock bit
175 bz %icc, 2f ! If unlocked, go displace
176 nop
177 sub %g3, (1 << 3), %g3
178 brgz %g3, 1b ! Still more TLB entries
179 nop ! to search
180
181 sethi %hi(sfmmu_panic5), %o0 ! We searched all entries and
182 call panic ! found no unlocked TTE so
183 or %o0, %lo(sfmmu_panic5), %o0 ! give up.
184
185
186 2:
187 !
188 ! We have found an unlocked or non-valid entry; we'll explicitly load
189 ! our locked entry here.
190 !
191 sethi %hi(FLUSH_ADDR), %o1 ! Flush addr doesn't matter
192 stxa %o0, [%o5]ASI_IMMU
193 stxa %g1, [%g3]ASI_ITLB_ACCESS
194 flush %o1 ! Flush required for I-MMU
195 ba 3f ! Delay slot of ba is empty
196 nop ! per Erratum 64
197
198 0:
199 #endif /* CHEETAHPLUS_ERRATUM_34 */
200 sethi %hi(FLUSH_ADDR), %o1 ! Flush addr doesn't matter
201 stxa %o0, [%o5]ASI_IMMU
202 stxa %g1, [%g0]ASI_ITLB_IN
203 flush %o1 ! Flush required for I-MMU
204 3:
205 retl
206 wrpr %g0, %o3, %pstate ! Enable interrupts
207 SET_SIZE(sfmmu_itlb_ld_kva)
208
209 /*
210 * Load an entry into the DTLB.
211 *
212 * Special handling is required for locked entries since there
213 * are some TLB slots that are reserved for the kernel but not
214 * always held locked. We want to avoid loading locked TTEs
215 * into those slots since they could be displaced.
216 *
217 * %o0 = virtual address
218 * %o1 = address of TTE to be loaded
219 */
220 ENTRY_NP(sfmmu_dtlb_ld_kva)
221 rdpr %pstate, %o3
222 #ifdef DEBUG
223 PANIC_IF_INTR_DISABLED_PSTR(%o3, msfmmu_di_l2, %g1)
224 #endif /* DEBUG */
225 wrpr %o3, PSTATE_IE, %pstate ! disable interrupts
226 srln %o0, MMU_PAGESHIFT, %o0
227 slln %o0, MMU_PAGESHIFT, %o0 ! clear page offset
228
229 ldx [%o1], %g1
230
231 set MMU_TAG_ACCESS, %o5
232
233 set cpu_impl_dual_pgsz, %o2
234 ld [%o2], %o2
235 brz %o2, 1f
236 nop
237
238 sethi %hi(ksfmmup), %o2
239 ldx [%o2 + %lo(ksfmmup)], %o2
240 ldub [%o2 + SFMMU_CEXT], %o2
241 sll %o2, TAGACCEXT_SHIFT, %o2
242
243 set MMU_TAG_ACCESS_EXT, %o4 ! can go into T8 if unlocked
244 stxa %o2,[%o4]ASI_DMMU
245 membar #Sync
246 1:
247 andcc %g1, TTE_LCK_INT, %g0 ! Locked entries require
248 bnz,pn %icc, 2f ! special handling
249 sethi %hi(dtlb_resv_ttenum), %g3
250 stxa %o0,[%o5]ASI_DMMU ! Load unlocked TTE
251 stxa %g1,[%g0]ASI_DTLB_IN ! via DTLB_IN
252 membar #Sync
253 retl
254 wrpr %g0, %o3, %pstate ! enable interrupts
255 2:
256 #ifdef CHEETAHPLUS_ERRATUM_34
257 GET_CPU_IMPL(%g2)
258 #endif
259 ld [%g3 + %lo(dtlb_resv_ttenum)], %g3
260 sll %g3, 3, %g3 ! First reserved idx in TLB 0
261 sub %g3, (1 << 3), %g3 ! Decrement idx
262 ! Erratum 15 workaround due to ld [%g3 + %lo(dtlb_resv_ttenum)], %g3
263 ldxa [%g3]ASI_DTLB_ACCESS, %g4 ! Load TTE from TLB 0
264 3:
265 ldxa [%g3]ASI_DTLB_ACCESS, %g4 ! Load TTE from TLB 0
266 !
267 ! If this entry isn't valid, we'll choose to displace it (regardless
268 ! of the lock bit).
269 !
270 brgez,pn %g4, 4f ! TTE is > 0 iff not valid
271 nop
272 andcc %g4, TTE_LCK_INT, %g0 ! Check for lock bit
273 bz,pn %icc, 4f ! If unlocked, go displace
274 nop
275 sub %g3, (1 << 3), %g3 ! Decrement idx
276 #ifdef CHEETAHPLUS_ERRATUM_34
277 !
278 ! If this is a Cheetah or derivative, we must work around Erratum 34
279 ! for the DTLB. Erratum 34 states that under certain conditions,
280 ! a locked entry 0 TTE may be improperly displaced. To avoid this,
281 ! we do not place a locked TTE in entry 0.
282 !
283 brgz %g3, 3b
284 nop
285 cmp %g2, CHEETAH_IMPL
286 bge,pt %icc, 5f
287 nop
288 brz %g3, 3b
289 nop
290 #else /* CHEETAHPLUS_ERRATUM_34 */
291 brgez %g3, 3b
292 nop
293 #endif /* CHEETAHPLUS_ERRATUM_34 */
294 5:
295 sethi %hi(sfmmu_panic5), %o0 ! We searched all entries and
296 call panic ! found no unlocked TTE so
297 or %o0, %lo(sfmmu_panic5), %o0 ! give up.
298 4:
299 stxa %o0,[%o5]ASI_DMMU ! Setup tag access
300 #ifdef OLYMPUS_SHARED_FTLB
301 stxa %g1,[%g0]ASI_DTLB_IN
302 #else
303 stxa %g1,[%g3]ASI_DTLB_ACCESS ! Displace entry at idx
304 #endif
305 membar #Sync
306 retl
307 wrpr %g0, %o3, %pstate ! enable interrupts
308 SET_SIZE(sfmmu_dtlb_ld_kva)
309
310 ENTRY_NP(sfmmu_getctx_pri)
311 set MMU_PCONTEXT, %o0
312 retl
313 ldxa [%o0]ASI_MMU_CTX, %o0
314 SET_SIZE(sfmmu_getctx_pri)
315
316 ENTRY_NP(sfmmu_getctx_sec)
317 set MMU_SCONTEXT, %o0
318 set CTXREG_CTX_MASK, %o1
319 ldxa [%o0]ASI_MMU_CTX, %o0
320 retl
321 and %o0, %o1, %o0
322 SET_SIZE(sfmmu_getctx_sec)
323
324 /*
325 * Set the secondary context register for this process.
326 * %o0 = page_size | context number for this process.
327 */
328 ENTRY_NP(sfmmu_setctx_sec)
329 /*
330 * From resume we call sfmmu_setctx_sec with interrupts disabled.
331 * But we can also get called from C with interrupts enabled. So,
332 * we need to check first.
333 */
334
335 /* If interrupts are not disabled, then disable them */
336 rdpr %pstate, %g1
337 btst PSTATE_IE, %g1
338 bnz,a,pt %icc, 1f
339 wrpr %g1, PSTATE_IE, %pstate /* disable interrupts */
340
341 1:
342 mov MMU_SCONTEXT, %o1
343
344 sethi %hi(FLUSH_ADDR), %o4
345 stxa %o0, [%o1]ASI_MMU_CTX /* set 2nd context reg. */
346 flush %o4
347 sethi %hi(shctx_on), %g3
348 ld [%g3 + %lo(shctx_on)], %g3
349 brz %g3, 2f
350 nop
351 set CTXREG_CTX_MASK, %o4
352 and %o0,%o4,%o1
353 cmp %o1, INVALID_CONTEXT
354 bne,pn %icc, 2f
355 mov MMU_SHARED_CONTEXT, %o1
356 sethi %hi(FLUSH_ADDR), %o4
357 stxa %g0, [%o1]ASI_MMU_CTX /* set 2nd context reg. */
358 flush %o4
359
360 /*
361 * if the routine was entered with intr enabled, then enable intr now.
362 * otherwise, keep intr disabled, return without enabing intr.
363 * %g1 - old intr state
364 */
365 2: btst PSTATE_IE, %g1
366 bnz,a,pt %icc, 3f
367 wrpr %g0, %g1, %pstate /* enable interrupts */
368 3: retl
369 nop
370 SET_SIZE(sfmmu_setctx_sec)
371
372 /*
373 * set ktsb_phys to 1 if the processor supports ASI_QUAD_LDD_PHYS.
374 * returns the detection value in %o0.
375 *
376 * Currently ASI_QUAD_LDD_PHYS is supported in processors as follows
377 * - cheetah+ and later (greater or equal to CHEETAH_PLUS_IMPL)
378 * - FJ OPL Olympus-C and later (less than SPITFIRE_IMPL)
379 *
380 */
381 ENTRY_NP(sfmmu_setup_4lp)
382 GET_CPU_IMPL(%o0);
383 cmp %o0, CHEETAH_PLUS_IMPL
384 bge,pt %icc, 4f
385 mov 1, %o1
386 cmp %o0, SPITFIRE_IMPL
387 bge,a,pn %icc, 3f
388 clr %o1
389 4:
390 set ktsb_phys, %o2
391 st %o1, [%o2]
392 3: retl
393 mov %o1, %o0
394 SET_SIZE(sfmmu_setup_4lp)
395
396
397 /*
398 * Called to load MMU registers and tsbmiss area
399 * for the active process. This function should
400 * only be called from TL=0.
401 *
402 * %o0 - hat pointer
403 *
404 */
405 ENTRY_NP(sfmmu_load_mmustate)
406
407 #ifdef DEBUG
408 PANIC_IF_INTR_ENABLED_PSTR(msfmmu_ei_l3, %g1)
409 #endif /* DEBUG */
410
411 sethi %hi(ksfmmup), %o3
412 ldx [%o3 + %lo(ksfmmup)], %o3
413 cmp %o3, %o0
414 be,pn %xcc, 8f ! if kernel as, do nothing
415 nop
416 /*
417 * We need to set up the TSB base register, tsbmiss
418 * area, and load locked TTE(s) for the TSB.
419 */
420 ldx [%o0 + SFMMU_TSB], %o1 ! %o1 = first tsbinfo
421 ldx [%o1 + TSBINFO_NEXTPTR], %g2 ! %g2 = second tsbinfo
422
423 #ifdef UTSB_PHYS
424 /*
425 * UTSB_PHYS accesses user TSBs via physical addresses. The first
426 * TSB is in the MMU I/D TSB Base registers. The 2nd, 3rd and
427 * 4th TSBs use designated ASI_SCRATCHPAD regs as pseudo TSB base regs.
428 */
429
430 /* create/set first UTSBREG actually loaded into MMU_TSB */
431 MAKE_UTSBREG(%o1, %o2, %o3) ! %o2 = first utsbreg
432 LOAD_TSBREG(%o2, %o3, %o4) ! write TSB base register
433
434 brz,a,pt %g2, 2f
435 mov -1, %o2 ! use -1 if no second TSB
436
437 MAKE_UTSBREG(%g2, %o2, %o3) ! %o2 = second utsbreg
438 2:
439 SET_UTSBREG(SCRATCHPAD_UTSBREG2, %o2, %o3)
440
441 /* make 3rd and 4th TSB */
442 CPU_TSBMISS_AREA(%o4, %o3) ! %o4 = tsbmiss area
443
444 ldx [%o0 + SFMMU_SCDP], %g2 ! %g2 = sfmmu_scd
445 brz,pt %g2, 3f
446 mov -1, %o2 ! use -1 if no third TSB
447
448 ldx [%g2 + SCD_SFMMUP], %g3 ! %g3 = scdp->scd_sfmmup
449 ldx [%g3 + SFMMU_TSB], %o1 ! %o1 = first scd tsbinfo
450 brz,pn %o1, 5f
451 nop ! panic if no third TSB
452
453 /* make 3rd UTSBREG */
454 MAKE_UTSBREG(%o1, %o2, %o3) ! %o2 = third utsbreg
455 3:
456 SET_UTSBREG(SCRATCHPAD_UTSBREG3, %o2, %o3)
457 stn %o2, [%o4 + TSBMISS_TSBSCDPTR]
458
459 brz,pt %g2, 4f
460 mov -1, %o2 ! use -1 if no 3rd or 4th TSB
461
462 ldx [%o1 + TSBINFO_NEXTPTR], %g2 ! %g2 = second scd tsbinfo
463 brz,pt %g2, 4f
464 mov -1, %o2 ! use -1 if no 4th TSB
465
466 /* make 4th UTSBREG */
467 MAKE_UTSBREG(%g2, %o2, %o3) ! %o2 = fourth utsbreg
468 4:
469 SET_UTSBREG(SCRATCHPAD_UTSBREG4, %o2, %o3)
470 stn %o2, [%o4 + TSBMISS_TSBSCDPTR4M]
471 ba,pt %icc, 6f
472 mov %o4, %o2 ! %o2 = tsbmiss area
473 5:
474 sethi %hi(panicstr), %g1 ! panic if no 3rd TSB
475 ldx [%g1 + %lo(panicstr)], %g1
476 tst %g1
477
478 bnz,pn %xcc, 8f
479 nop
480
481 sethi %hi(sfmmu_panic10), %o0
482 call panic
483 or %o0, %lo(sfmmu_panic10), %o0
484
485 #else /* UTSBREG_PHYS */
486
487 brz,pt %g2, 4f
488 nop
489 /*
490 * We have a second TSB for this process, so we need to
491 * encode data for both the first and second TSB in our single
492 * TSB base register. See hat_sfmmu.h for details on what bits
493 * correspond to which TSB.
494 * We also need to load a locked TTE into the TLB for the second TSB
495 * in this case.
496 */
497 MAKE_TSBREG_SECTSB(%o2, %o1, %g2, %o3, %o4, %g3, sfmmu_tsb_2nd)
498 ! %o2 = tsbreg
499 sethi %hi(utsb4m_dtlb_ttenum), %o3
500 sethi %hi(utsb4m_vabase), %o4
501 ld [%o3 + %lo(utsb4m_dtlb_ttenum)], %o3
502 ldx [%o4 + %lo(utsb4m_vabase)], %o4 ! %o4 = TLB tag for sec TSB
503 sll %o3, DTACC_SHIFT, %o3 ! %o3 = sec TSB TLB index
504 RESV_OFFSET(%g2, %o4, %g3, sfmmu_tsb_2nd) ! or-in bits of TSB VA
505 LOAD_TSBTTE(%g2, %o3, %o4, %g3) ! load sec TSB locked TTE
506 sethi %hi(utsb_vabase), %g3
507 ldx [%g3 + %lo(utsb_vabase)], %g3 ! %g3 = TLB tag for first TSB
508 ba,pt %xcc, 5f
509 nop
510
511 4: sethi %hi(utsb_vabase), %g3
512 ldx [%g3 + %lo(utsb_vabase)], %g3 ! %g3 = TLB tag for first TSB
513 MAKE_TSBREG(%o2, %o1, %g3, %o3, %o4, sfmmu_tsb_1st) ! %o2 = tsbreg
514
515 5: LOAD_TSBREG(%o2, %o3, %o4) ! write TSB base register
516
517 /*
518 * Load the TTE for the first TSB at the appropriate location in
519 * the TLB
520 */
521 sethi %hi(utsb_dtlb_ttenum), %o2
522 ld [%o2 + %lo(utsb_dtlb_ttenum)], %o2
523 sll %o2, DTACC_SHIFT, %o2 ! %o1 = first TSB TLB index
524 RESV_OFFSET(%o1, %g3, %o3, sfmmu_tsb_1st) ! or-in bits of TSB VA
525 LOAD_TSBTTE(%o1, %o2, %g3, %o4) ! load first TSB locked TTE
526 CPU_TSBMISS_AREA(%o2, %o3)
527 #endif /* UTSB_PHYS */
528 6:
529 ldx [%o0 + SFMMU_ISMBLKPA], %o1 ! copy members of sfmmu
530 ! we need to access from
531 stx %o1, [%o2 + TSBMISS_ISMBLKPA] ! sfmmu_tsb_miss into the
532 ldub [%o0 + SFMMU_TTEFLAGS], %o3 ! per-CPU tsbmiss area.
533 stx %o0, [%o2 + TSBMISS_UHATID]
534 stub %o3, [%o2 + TSBMISS_UTTEFLAGS]
535 #ifdef UTSB_PHYS
536 ldx [%o0 + SFMMU_SRDP], %o1
537 ldub [%o0 + SFMMU_RTTEFLAGS], %o4
538 stub %o4, [%o2 + TSBMISS_URTTEFLAGS]
539 stx %o1, [%o2 + TSBMISS_SHARED_UHATID]
540 brz,pn %o1, 8f ! check for sfmmu_srdp
541 add %o0, SFMMU_HMERMAP, %o1
542 add %o2, TSBMISS_SHMERMAP, %o2
543 mov SFMMU_HMERGNMAP_WORDS, %o3
544 ! set tsbmiss shmermap
545 SET_REGION_MAP(%o1, %o2, %o3, %o4, load_shme_mmustate)
546
547 ldx [%o0 + SFMMU_SCDP], %o4 ! %o4 = sfmmu_scd
548 CPU_TSBMISS_AREA(%o2, %o3) ! %o2 = tsbmiss area
549 mov SFMMU_HMERGNMAP_WORDS, %o3
550 brnz,pt %o4, 7f ! check for sfmmu_scdp else
551 add %o2, TSBMISS_SCDSHMERMAP, %o2 ! zero tsbmiss scd_shmermap
552 ZERO_REGION_MAP(%o2, %o3, zero_scd_mmustate)
553 ba 8f
554 nop
555 7:
556 add %o4, SCD_HMERMAP, %o1
557 SET_REGION_MAP(%o1, %o2, %o3, %o4, load_scd_mmustate)
558 #endif /* UTSB_PHYS */
559
560 8:
561 retl
562 nop
563 SET_SIZE(sfmmu_load_mmustate)
564
565 /*
566 * Invalidate all of the entries within the TSB, by setting the inv bit
567 * in the tte_tag field of each tsbe.
568 *
569 * We take advantage of the fact that the TSBs are page aligned and a
570 * multiple of PAGESIZE to use ASI_BLK_INIT_xxx ASI.
571 *
572 * See TSB_LOCK_ENTRY and the miss handlers for how this works in practice
573 * (in short, we set all bits in the upper word of the tag, and we give the
574 * invalid bit precedence over other tag bits in both places).
575 */
576
577 #define VIS_BLOCKSIZE 64
578
579 ENTRY(sfmmu_inv_tsb_fast)
580
581 ! Get space for aligned block of saved fp regs.
582 save %sp, -SA(MINFRAME + 2*VIS_BLOCKSIZE), %sp
583
584 ! kpreempt_disable();
585 ldsb [THREAD_REG + T_PREEMPT], %l3
586 inc %l3
587 stb %l3, [THREAD_REG + T_PREEMPT]
588
589 ! See if fpu was in use. If it was, we need to save off the
590 ! floating point registers to the stack.
591 rd %fprs, %l0 ! %l0 = cached copy of fprs
592 btst FPRS_FEF, %l0
593 bz,pt %icc, 4f
594 nop
595
596 ! save in-use fpregs on stack
597 membar #Sync ! make sure tranx to fp regs
598 ! have completed
599 add %fp, STACK_BIAS - 65, %l1 ! get stack frame for fp regs
600 and %l1, -VIS_BLOCKSIZE, %l1 ! block align frame
601 stda %d0, [%l1]ASI_BLK_P ! %l1 = addr of saved fp regs
602
603 ! enable fp
604 4: membar #StoreStore|#StoreLoad|#LoadStore
605 wr %g0, FPRS_FEF, %fprs
606 wr %g0, ASI_BLK_P, %asi
607
608 ! load up FP registers with invalid TSB tag.
609 fone %d0 ! ones in tag
610 fzero %d2 ! zeros in TTE
611 fone %d4 ! ones in tag
612 fzero %d6 ! zeros in TTE
613 fone %d8 ! ones in tag
614 fzero %d10 ! zeros in TTE
615 fone %d12 ! ones in tag
616 fzero %d14 ! zeros in TTE
617 ba,pt %xcc, .sfmmu_inv_doblock
618 mov (4*VIS_BLOCKSIZE), %i4 ! we do 4 stda's each loop below
619
620 .sfmmu_inv_blkstart:
621 ! stda %d0, [%i0+192]%asi ! in dly slot of branch that got us here
622 stda %d0, [%i0+128]%asi
623 stda %d0, [%i0+64]%asi
624 stda %d0, [%i0]%asi
625
626 add %i0, %i4, %i0
627 sub %i1, %i4, %i1
628
629 .sfmmu_inv_doblock:
630 cmp %i1, (4*VIS_BLOCKSIZE) ! check for completion
631 bgeu,a %icc, .sfmmu_inv_blkstart
632 stda %d0, [%i0+192]%asi
633
634 .sfmmu_inv_finish:
635 membar #Sync
636 btst FPRS_FEF, %l0 ! saved from above
637 bz,a .sfmmu_inv_finished
638 wr %l0, 0, %fprs ! restore fprs
639
640 ! restore fpregs from stack
641 ldda [%l1]ASI_BLK_P, %d0
642 membar #Sync
643 wr %l0, 0, %fprs ! restore fprs
644
645 .sfmmu_inv_finished:
646 ! kpreempt_enable();
647 ldsb [THREAD_REG + T_PREEMPT], %l3
648 dec %l3
649 stb %l3, [THREAD_REG + T_PREEMPT]
650 ret
651 restore
652 SET_SIZE(sfmmu_inv_tsb_fast)
653
654 /*
655 * Prefetch "struct tsbe" while walking TSBs.
656 * prefetch 7 cache lines ahead of where we are at now.
657 * #n_reads is being used since #one_read only applies to
658 * floating point reads, and we are not doing floating point
659 * reads. However, this has the negative side effect of polluting
660 * the ecache.
661 * The 448 comes from (7 * 64) which is how far ahead of our current
662 * address, we want to prefetch.
663 */
664 ENTRY(prefetch_tsbe_read)
665 retl
666 prefetch [%o0+448], #n_reads
667 SET_SIZE(prefetch_tsbe_read)
668
669 /* Prefetch the tsbe that we are about to write */
670 ENTRY(prefetch_tsbe_write)
671 retl
672 prefetch [%o0], #n_writes
673 SET_SIZE(prefetch_tsbe_write)
674