1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 /*
28 * SFMMU primitives. These primitives should only be used by sfmmu
29 * routines.
30 */
31
32 #if defined(lint)
33 #include <sys/types.h>
34 #else /* lint */
35 #include "assym.h"
36 #endif /* lint */
37
38 #include <sys/asm_linkage.h>
39 #include <sys/machtrap.h>
40 #include <sys/machasi.h>
41 #include <sys/sun4asi.h>
42 #include <sys/pte.h>
43 #include <sys/mmu.h>
44 #include <vm/hat_sfmmu.h>
45 #include <vm/seg_spt.h>
46 #include <sys/machparam.h>
47 #include <sys/privregs.h>
48 #include <sys/scb.h>
49 #include <sys/intreg.h>
50 #include <sys/machthread.h>
51 #include <sys/intr.h>
52 #include <sys/clock.h>
53 #include <sys/trapstat.h>
54
55 #ifdef TRAPTRACE
56 #include <sys/traptrace.h>
57
58 /*
59 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
60 */
61 #define TT_TRACE(label) \
62 ba label ;\
63 rd %pc, %g7
64 #else
65
66 #define TT_TRACE(label)
67
68 #endif /* TRAPTRACE */
69
70 #ifndef lint
71
72 #if (TTE_SUSPEND_SHIFT > 0)
73 #define TTE_SUSPEND_INT_SHIFT(reg) \
74 sllx reg, TTE_SUSPEND_SHIFT, reg
75 #else
76 #define TTE_SUSPEND_INT_SHIFT(reg)
77 #endif
78
79 #endif /* lint */
80
81 #ifndef lint
82
83 /*
84 * Assumes TSBE_TAG is 0
85 * Assumes TSBE_INTHI is 0
86 * Assumes TSBREG.split is 0
87 */
88
89 #if TSBE_TAG != 0
90 #error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
91 #endif
92
93 #if TSBTAG_INTHI != 0
94 #error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
95 #endif
96
97 /*
98 * The following code assumes the tsb is not split.
99 *
100 * With TSBs no longer shared between processes, it's no longer
101 * necessary to hash the context bits into the tsb index to get
102 * tsb coloring; the new implementation treats the TSB as a
103 * direct-mapped, virtually-addressed cache.
104 *
105 * In:
106 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
107 * tsbbase = base address of TSB (clobbered)
108 * tagacc = tag access register (clobbered)
109 * szc = size code of TSB (ro)
110 * tmp = scratch reg
111 * Out:
112 * tsbbase = pointer to entry in TSB
113 */
114 #define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \
115 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\
116 srlx tagacc, vpshift, tagacc ;\
117 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\
118 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\
119 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\
120 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\
121 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */
122
123 /*
124 * When the kpm TSB is used it is assumed that it is direct mapped
125 * using (vaddr>>vpshift)%tsbsz as the index.
126 *
127 * Note that, for now, the kpm TSB and kernel TSB are the same for
128 * each mapping size. However that need not always be the case. If
129 * the trap handlers are updated to search a different TSB for kpm
130 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
131 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
132 *
133 * In:
134 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
135 * vaddr = virtual address (clobbered)
136 * tsbp, szc, tmp = scratch
137 * Out:
138 * tsbp = pointer to entry in TSB
139 */
140 #define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \
141 cmp vpshift, MMU_PAGESHIFT ;\
142 bne,pn %icc, 1f /* branch if large case */ ;\
143 sethi %hi(kpmsm_tsbsz), szc ;\
144 sethi %hi(kpmsm_tsbbase), tsbp ;\
145 ld [szc + %lo(kpmsm_tsbsz)], szc ;\
146 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\
147 ba,pt %icc, 2f ;\
148 nop ;\
149 1: sethi %hi(kpm_tsbsz), szc ;\
150 sethi %hi(kpm_tsbbase), tsbp ;\
151 ld [szc + %lo(kpm_tsbsz)], szc ;\
152 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\
153 2: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
154
155 /*
156 * Lock the TSBE at virtual address tsbep.
157 *
158 * tsbep = TSBE va (ro)
159 * tmp1, tmp2 = scratch registers (clobbered)
160 * label = label to jump to if we fail to lock the tsb entry
161 * %asi = ASI to use for TSB access
162 *
163 * NOTE that we flush the TSB using fast VIS instructions that
164 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
165 * not be treated as a locked entry or we'll get stuck spinning on
166 * an entry that isn't locked but really invalid.
167 */
168
169 #if defined(UTSB_PHYS)
170
171 #define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \
172 lda [tsbep]ASI_MEM, tmp1 ;\
173 sethi %hi(TSBTAG_LOCKED), tmp2 ;\
174 cmp tmp1, tmp2 ;\
175 be,a,pn %icc, label /* if locked ignore */ ;\
176 nop ;\
177 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\
178 cmp tmp1, tmp2 ;\
179 bne,a,pn %icc, label /* didn't lock so ignore */ ;\
180 nop ;\
181 /* tsbe lock acquired */ ;\
182 membar #StoreStore
183
184 #else /* UTSB_PHYS */
185
186 #define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \
187 lda [tsbep]%asi, tmp1 ;\
188 sethi %hi(TSBTAG_LOCKED), tmp2 ;\
189 cmp tmp1, tmp2 ;\
190 be,a,pn %icc, label /* if locked ignore */ ;\
191 nop ;\
192 casa [tsbep]%asi, tmp1, tmp2 ;\
193 cmp tmp1, tmp2 ;\
194 bne,a,pn %icc, label /* didn't lock so ignore */ ;\
195 nop ;\
196 /* tsbe lock acquired */ ;\
197 membar #StoreStore
198
199 #endif /* UTSB_PHYS */
200
201 /*
202 * Atomically write TSBE at virtual address tsbep.
203 *
204 * tsbep = TSBE va (ro)
205 * tte = TSBE TTE (ro)
206 * tagtarget = TSBE tag (ro)
207 * %asi = ASI to use for TSB access
208 */
209
210 #if defined(UTSB_PHYS)
211
212 #define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \
213 add tsbep, TSBE_TTE, tmp1 ;\
214 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\
215 membar #StoreStore ;\
216 add tsbep, TSBE_TAG, tmp1 ;\
217 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */
218
219 #else /* UTSB_PHYS */
220
221 #define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \
222 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\
223 membar #StoreStore ;\
224 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
225
226 #endif /* UTSB_PHYS */
227
228 /*
229 * Load an entry into the TSB at TL > 0.
230 *
231 * tsbep = pointer to the TSBE to load as va (ro)
232 * tte = value of the TTE retrieved and loaded (wo)
233 * tagtarget = tag target register. To get TSBE tag to load,
234 * we need to mask off the context and leave only the va (clobbered)
235 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
236 * tmp1, tmp2 = scratch registers
237 * label = label to jump to if we fail to lock the tsb entry
238 * %asi = ASI to use for TSB access
239 */
240
241 #if defined(UTSB_PHYS)
242
243 #define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
244 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\
245 /* ;\
246 * I don't need to update the TSB then check for the valid tte. ;\
247 * TSB invalidate will spin till the entry is unlocked. Note, ;\
248 * we always invalidate the hash table before we unload the TSB.;\
249 */ ;\
250 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\
251 ldxa [ttepa]ASI_MEM, tte ;\
252 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\
253 sethi %hi(TSBTAG_INVALID), tmp2 ;\
254 add tsbep, TSBE_TAG, tmp1 ;\
255 brgez,a,pn tte, label ;\
256 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\
257 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\
258 label:
259
260 #else /* UTSB_PHYS */
261
262 #define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
263 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\
264 /* ;\
265 * I don't need to update the TSB then check for the valid tte. ;\
266 * TSB invalidate will spin till the entry is unlocked. Note, ;\
267 * we always invalidate the hash table before we unload the TSB.;\
268 */ ;\
269 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\
270 ldxa [ttepa]ASI_MEM, tte ;\
271 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\
272 sethi %hi(TSBTAG_INVALID), tmp2 ;\
273 brgez,a,pn tte, label ;\
274 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\
275 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\
276 label:
277
278 #endif /* UTSB_PHYS */
279
280 /*
281 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
282 * for ITLB synthesis.
283 *
284 * tsbep = pointer to the TSBE to load as va (ro)
285 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
286 * with exec_perm turned off and exec_synth turned on
287 * tagtarget = tag target register. To get TSBE tag to load,
288 * we need to mask off the context and leave only the va (clobbered)
289 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
290 * tmp1, tmp2 = scratch registers
291 * label = label to use for branch (text)
292 * %asi = ASI to use for TSB access
293 */
294
295 #define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
296 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\
297 /* ;\
298 * I don't need to update the TSB then check for the valid tte. ;\
299 * TSB invalidate will spin till the entry is unlocked. Note, ;\
300 * we always invalidate the hash table before we unload the TSB.;\
301 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\
302 * and exec_synth bit to 1. ;\
303 */ ;\
304 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\
305 mov tte, tmp1 ;\
306 ldxa [ttepa]ASI_MEM, tte ;\
307 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\
308 sethi %hi(TSBTAG_INVALID), tmp2 ;\
309 brgez,a,pn tte, label ;\
310 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\
311 or tte, tmp1, tte ;\
312 andn tte, TTE_EXECPRM_INT, tte ;\
313 or tte, TTE_E_SYNTH_INT, tte ;\
314 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\
315 label:
316
317 /*
318 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
319 *
320 * tte = value of the TTE, used to get tte_size bits (ro)
321 * tagaccess = tag access register, used to get 4M pfn bits (ro)
322 * pfn = 4M pfn bits shifted to offset for tte (out)
323 * tmp1 = scratch register
324 * label = label to use for branch (text)
325 */
326
327 #define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \
328 /* ;\
329 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\
330 * Return them, shifted, in pfn. ;\
331 */ ;\
332 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\
333 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\
334 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\
335 bz,a,pt %icc, label/**/f /* if 0, is */ ;\
336 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\
337 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\
338 label: ;\
339 sllx tagaccess, MMU_PAGESHIFT4M, pfn
340
341 /*
342 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
343 * for ITLB synthesis.
344 *
345 * tte = value of the TTE, used to get tte_size bits (rw)
346 * tmp1 = scratch register
347 */
348
349 #define SET_TTE4M_PN(tte, tmp) \
350 /* ;\
351 * Set 4M pagesize tte bits. ;\
352 */ ;\
353 set TTE4M, tmp ;\
354 sllx tmp, TTE_SZ_SHFT, tmp ;\
355 or tte, tmp, tte
356
357 /*
358 * Load an entry into the TSB at TL=0.
359 *
360 * tsbep = pointer to the TSBE to load as va (ro)
361 * tteva = pointer to the TTE to load as va (ro)
362 * tagtarget = TSBE tag to load (which contains no context), synthesized
363 * to match va of MMU tag target register only (ro)
364 * tmp1, tmp2 = scratch registers (clobbered)
365 * label = label to use for branches (text)
366 * %asi = ASI to use for TSB access
367 */
368
369 #if defined(UTSB_PHYS)
370
371 #define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \
372 /* can't rd tteva after locking tsb because it can tlb miss */ ;\
373 ldx [tteva], tteva /* load tte */ ;\
374 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\
375 sethi %hi(TSBTAG_INVALID), tmp2 ;\
376 add tsbep, TSBE_TAG, tmp1 ;\
377 brgez,a,pn tteva, label ;\
378 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\
379 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\
380 label:
381
382 #else /* UTSB_PHYS */
383
384 #define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \
385 /* can't rd tteva after locking tsb because it can tlb miss */ ;\
386 ldx [tteva], tteva /* load tte */ ;\
387 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\
388 sethi %hi(TSBTAG_INVALID), tmp2 ;\
389 brgez,a,pn tteva, label ;\
390 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\
391 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\
392 label:
393
394 #endif /* UTSB_PHYS */
395
396 /*
397 * Invalidate a TSB entry in the TSB.
398 *
399 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check
400 * about this earlier to ensure this is true. Thus when we are
401 * directly referencing tsbep below, we are referencing the tte_tag
402 * field of the TSBE. If this offset ever changes, the code below
403 * will need to be modified.
404 *
405 * tsbep = pointer to TSBE as va (ro)
406 * tag = invalidation is done if this matches the TSBE tag (ro)
407 * tmp1 - tmp3 = scratch registers (clobbered)
408 * label = label name to use for branches (text)
409 * %asi = ASI to use for TSB access
410 */
411
412 #if defined(UTSB_PHYS)
413
414 #define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \
415 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\
416 sethi %hi(TSBTAG_LOCKED), tmp2 ;\
417 label/**/1: ;\
418 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\
419 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\
420 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\
421 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\
422 cmp tag, tmp3 /* compare tags */ ;\
423 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\
424 sethi %hi(TSBTAG_INVALID), tmp3 ;\
425 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\
426 cmp tmp1, tmp3 /* if not successful */ ;\
427 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\
428 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\
429 label/**/2:
430
431 #else /* UTSB_PHYS */
432
433 #define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \
434 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\
435 sethi %hi(TSBTAG_LOCKED), tmp2 ;\
436 label/**/1: ;\
437 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\
438 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\
439 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\
440 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\
441 cmp tag, tmp3 /* compare tags */ ;\
442 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\
443 sethi %hi(TSBTAG_INVALID), tmp3 ;\
444 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\
445 cmp tmp1, tmp3 /* if not successful */ ;\
446 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\
447 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\
448 label/**/2:
449
450 #endif /* UTSB_PHYS */
451
452 #if TSB_SOFTSZ_MASK < TSB_SZ_MASK
453 #error - TSB_SOFTSZ_MASK too small
454 #endif
455
456
457 /*
458 * An implementation of setx which will be hot patched at run time.
459 * since it is being hot patched, there is no value passed in.
460 * Thus, essentially we are implementing
461 * setx value, tmp, dest
462 * where value is RUNTIME_PATCH (aka 0) in this case.
463 */
464 #define RUNTIME_PATCH_SETX(dest, tmp) \
465 sethi %hh(RUNTIME_PATCH), tmp ;\
466 sethi %lm(RUNTIME_PATCH), dest ;\
467 or tmp, %hm(RUNTIME_PATCH), tmp ;\
468 or dest, %lo(RUNTIME_PATCH), dest ;\
469 sllx tmp, 32, tmp ;\
470 nop /* for perf reasons */ ;\
471 or tmp, dest, dest /* contents of patched value */
472
473 #endif /* lint */
474
475
476 #if defined (lint)
477
478 /*
479 * sfmmu related subroutines
480 */
481 uint_t
482 sfmmu_disable_intrs()
483 { return(0); }
484
485 /* ARGSUSED */
486 void
487 sfmmu_enable_intrs(uint_t pstate_save)
488 {}
489
490 /* ARGSUSED */
491 int
492 sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag)
493 { return(0); }
494
495 /*
496 * Use cas, if tte has changed underneath us then reread and try again.
497 * In the case of a retry, it will update sttep with the new original.
498 */
499 /* ARGSUSED */
500 int
501 sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
502 { return(0); }
503
504 /*
505 * Use cas, if tte has changed underneath us then return 1, else return 0
506 */
507 /* ARGSUSED */
508 int
509 sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
510 { return(0); }
511
512 /* ARGSUSED */
513 void
514 sfmmu_copytte(tte_t *sttep, tte_t *dttep)
515 {}
516
517 /*ARGSUSED*/
518 struct tsbe *
519 sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
520 { return(0); }
521
522 /*ARGSUSED*/
523 uint64_t
524 sfmmu_make_tsbtag(caddr_t va)
525 { return(0); }
526
527 #else /* lint */
528
529 .seg ".data"
530 .global sfmmu_panic1
531 sfmmu_panic1:
532 .asciz "sfmmu_asm: interrupts already disabled"
533
534 .global sfmmu_panic3
535 sfmmu_panic3:
536 .asciz "sfmmu_asm: sfmmu_vatopfn called for user"
537
538 .global sfmmu_panic4
539 sfmmu_panic4:
540 .asciz "sfmmu_asm: 4M tsb pointer mis-match"
541
542 .global sfmmu_panic5
543 sfmmu_panic5:
544 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0"
545
546 .global sfmmu_panic6
547 sfmmu_panic6:
548 .asciz "sfmmu_asm: interrupts not disabled"
549
550 .global sfmmu_panic7
551 sfmmu_panic7:
552 .asciz "sfmmu_asm: kernel as"
553
554 .global sfmmu_panic8
555 sfmmu_panic8:
556 .asciz "sfmmu_asm: gnum is zero"
557
558 .global sfmmu_panic9
559 sfmmu_panic9:
560 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
561
562 .global sfmmu_panic10
563 sfmmu_panic10:
564 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB"
565
566 .global sfmmu_panic11
567 sfmmu_panic11:
568 .asciz "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform"
569
570 ENTRY(sfmmu_disable_intrs)
571 rdpr %pstate, %o0
572 #ifdef DEBUG
573 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
574 #endif /* DEBUG */
575 retl
576 wrpr %o0, PSTATE_IE, %pstate
577 SET_SIZE(sfmmu_disable_intrs)
578
579 ENTRY(sfmmu_enable_intrs)
580 retl
581 wrpr %g0, %o0, %pstate
582 SET_SIZE(sfmmu_enable_intrs)
583
584 /*
585 * This routine is called both by resume() and sfmmu_get_ctx() to
586 * allocate a new context for the process on a MMU.
587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
589 * is the case when sfmmu_alloc_ctx is called from resume().
590 *
591 * The caller must disable interrupts before entering this routine.
592 * To reduce ctx switch overhead, the code contains both 'fast path' and
593 * 'slow path' code. The fast path code covers the common case where only
594 * a quick check is needed and the real ctx allocation is not required.
595 * It can be done without holding the per-process (PP) lock.
596 * The 'slow path' code must be protected by the PP Lock and performs ctx
597 * allocation.
598 * Hardware context register and HAT mmu cnum are updated accordingly.
599 *
600 * %o0 - sfmmup
601 * %o1 - allocflag
602 * %o2 - CPU
603 * %o3 - sfmmu private/shared flag
604 *
605 * ret - 0: no ctx is allocated
606 * 1: a ctx is allocated
607 */
608 ENTRY_NP(sfmmu_alloc_ctx)
609
610 #ifdef DEBUG
611 sethi %hi(ksfmmup), %g1
612 ldx [%g1 + %lo(ksfmmup)], %g1
613 cmp %g1, %o0
614 bne,pt %xcc, 0f
615 nop
616
617 sethi %hi(panicstr), %g1 ! if kernel as, panic
618 ldx [%g1 + %lo(panicstr)], %g1
619 tst %g1
620 bnz,pn %icc, 7f
621 nop
622
623 sethi %hi(sfmmu_panic7), %o0
624 call panic
625 or %o0, %lo(sfmmu_panic7), %o0
626
627 7:
628 retl
629 mov %g0, %o0 ! %o0 = ret = 0
630
631 0:
632 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
633 #endif /* DEBUG */
634
635 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1
636
637 ! load global mmu_ctxp info
638 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr
639
640 #ifdef sun4v
641 /* During suspend on sun4v, context domains can be temporary removed */
642 brz,a,pn %o3, 0f
643 nop
644 #endif
645
646 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index
647
648 ! load global mmu_ctxp gnum
649 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum
650
651 #ifdef DEBUG
652 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0
653 bne,pt %xcc, 3f
654 nop
655
656 sethi %hi(panicstr), %g1 ! test if panicstr is already set
657 ldx [%g1 + %lo(panicstr)], %g1
658 tst %g1
659 bnz,pn %icc, 1f
660 nop
661
662 sethi %hi(sfmmu_panic8), %o0
663 call panic
664 or %o0, %lo(sfmmu_panic8), %o0
665 1:
666 retl
667 mov %g0, %o0 ! %o0 = ret = 0
668 3:
669 #endif
670
671 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum
672
673 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2
674 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
675
676 /*
677 * %g5 = sfmmu gnum returned
678 * %g6 = sfmmu cnum returned
679 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
680 * %g4 = scratch
681 *
682 * Fast path code, do a quick check.
683 */
684 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
685
686 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ??
687 bne,pt %icc, 1f ! valid hat cnum, check gnum
688 nop
689
690 ! cnum == INVALID, check allocflag
691 mov %g0, %g4 ! %g4 = ret = 0
692 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail
693 mov %g6, %o1
694
695 ! (invalid HAT cnum) && (allocflag == 1)
696 ba,pt %icc, 2f
697 nop
698 #ifdef sun4v
699 0:
700 set INVALID_CONTEXT, %o1
701 membar #LoadStore|#StoreStore
702 ba,pt %icc, 8f
703 mov %g0, %g4 ! %g4 = ret = 0
704 #endif
705 1:
706 ! valid HAT cnum, check gnum
707 cmp %g5, %o4
708 mov 1, %g4 !%g4 = ret = 1
709 be,a,pt %icc, 8f ! gnum unchanged, go to done
710 mov %g6, %o1
711
712 2:
713 /*
714 * Grab per process (PP) sfmmu_ctx_lock spinlock,
715 * followed by the 'slow path' code.
716 */
717 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock
718 3:
719 brz %g3, 5f
720 nop
721 4:
722 brnz,a,pt %g3, 4b ! spin if lock is 1
723 ldub [%o0 + SFMMU_CTX_LOCK], %g3
724 ba %xcc, 3b ! retry the lock
725 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock
726
727 5:
728 membar #LoadLoad
729 /*
730 * %g5 = sfmmu gnum returned
731 * %g6 = sfmmu cnum returned
732 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
733 * %g4 = scratch
734 */
735 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
736
737 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ??
738 bne,pt %icc, 1f ! valid hat cnum, check gnum
739 nop
740
741 ! cnum == INVALID, check allocflag
742 mov %g0, %g4 ! %g4 = ret = 0
743 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw
744 mov %g6, %o1
745
746 ! (invalid HAT cnum) && (allocflag == 1)
747 ba,pt %icc, 6f
748 nop
749 1:
750 ! valid HAT cnum, check gnum
751 cmp %g5, %o4
752 mov 1, %g4 ! %g4 = ret = 1
753 be,a,pt %icc, 2f ! gnum unchanged, go to done
754 mov %g6, %o1
755
756 ba,pt %icc, 6f
757 nop
758 2:
759 membar #LoadStore|#StoreStore
760 ba,pt %icc, 8f
761 clrb [%o0 + SFMMU_CTX_LOCK]
762 6:
763 /*
764 * We get here if we do not have a valid context, or
765 * the HAT gnum does not match global gnum. We hold
766 * sfmmu_ctx_lock spinlock. Allocate that context.
767 *
768 * %o3 = mmu_ctxp
769 */
770 add %o3, MMU_CTX_CNUM, %g3
771 ld [%o3 + MMU_CTX_NCTXS], %g4
772
773 /*
774 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
775 * %g3 = mmu cnum address
776 * %g4 = mmu nctxs
777 *
778 * %o0 = sfmmup
779 * %o1 = mmu current cnum value (used as new cnum)
780 * %o4 = mmu gnum
781 *
782 * %o5 = scratch
783 */
784 ld [%g3], %o1
785 0:
786 cmp %o1, %g4
787 bl,a,pt %icc, 1f
788 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1
789
790 /*
791 * cnum reachs max, bail, so wrap around can be performed later.
792 */
793 set INVALID_CONTEXT, %o1
794 mov %g0, %g4 ! %g4 = ret = 0
795
796 membar #LoadStore|#StoreStore
797 ba,pt %icc, 8f
798 clrb [%o0 + SFMMU_CTX_LOCK]
799 1:
800 ! %g3 = addr of mmu_ctxp->cnum
801 ! %o5 = mmu_ctxp->cnum + 1
802 cas [%g3], %o1, %o5
803 cmp %o1, %o5
804 bne,a,pn %xcc, 0b ! cas failed
805 ld [%g3], %o1
806
807 #ifdef DEBUG
808 set MAX_SFMMU_CTX_VAL, %o5
809 cmp %o1, %o5
810 ble,pt %icc, 2f
811 nop
812
813 sethi %hi(sfmmu_panic9), %o0
814 call panic
815 or %o0, %lo(sfmmu_panic9), %o0
816 2:
817 #endif
818 ! update hat gnum and cnum
819 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4
820 or %o4, %o1, %o4
821 stx %o4, [%g2 + SFMMU_CTXS]
822
823 membar #LoadStore|#StoreStore
824 clrb [%o0 + SFMMU_CTX_LOCK]
825
826 mov 1, %g4 ! %g4 = ret = 1
827 8:
828 /*
829 * program the secondary context register
830 *
831 * %o1 = cnum
832 * %g1 = sfmmu private/shared flag (0:private, 1:shared)
833 */
834
835 /*
836 * When we come here and context is invalid, we want to set both
837 * private and shared ctx regs to INVALID. In order to
838 * do so, we set the sfmmu priv/shared flag to 'private' regardless
839 * so that private ctx reg will be set to invalid.
840 * Note that on sun4v values written to private context register are
841 * automatically written to corresponding shared context register as
842 * well. On sun4u SET_SECCTX() will invalidate shared context register
843 * when it sets a private secondary context register.
844 */
845
846 cmp %o1, INVALID_CONTEXT
847 be,a,pn %icc, 9f
848 clr %g1
849 9:
850
851 #ifdef sun4u
852 ldub [%o0 + SFMMU_CEXT], %o2
853 sll %o2, CTXREG_EXT_SHIFT, %o2
854 or %o1, %o2, %o1
855 #endif /* sun4u */
856
857 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1)
858
859 retl
860 mov %g4, %o0 ! %o0 = ret
861
862 SET_SIZE(sfmmu_alloc_ctx)
863
864 ENTRY_NP(sfmmu_modifytte)
865 ldx [%o2], %g3 /* current */
866 ldx [%o0], %g1 /* original */
867 2:
868 ldx [%o1], %g2 /* modified */
869 cmp %g2, %g3 /* is modified = current? */
870 be,a,pt %xcc,1f /* yes, don't write */
871 stx %g3, [%o0] /* update new original */
872 casx [%o2], %g1, %g2
873 cmp %g1, %g2
874 be,pt %xcc, 1f /* cas succeeded - return */
875 nop
876 ldx [%o2], %g3 /* new current */
877 stx %g3, [%o0] /* save as new original */
878 ba,pt %xcc, 2b
879 mov %g3, %g1
880 1: retl
881 membar #StoreLoad
882 SET_SIZE(sfmmu_modifytte)
883
884 ENTRY_NP(sfmmu_modifytte_try)
885 ldx [%o1], %g2 /* modified */
886 ldx [%o2], %g3 /* current */
887 ldx [%o0], %g1 /* original */
888 cmp %g3, %g2 /* is modified = current? */
889 be,a,pn %xcc,1f /* yes, don't write */
890 mov 0, %o1 /* as if cas failed. */
891
892 casx [%o2], %g1, %g2
893 membar #StoreLoad
894 cmp %g1, %g2
895 movne %xcc, -1, %o1 /* cas failed. */
896 move %xcc, 1, %o1 /* cas succeeded. */
897 1:
898 stx %g2, [%o0] /* report "current" value */
899 retl
900 mov %o1, %o0
901 SET_SIZE(sfmmu_modifytte_try)
902
903 ENTRY_NP(sfmmu_copytte)
904 ldx [%o0], %g1
905 retl
906 stx %g1, [%o1]
907 SET_SIZE(sfmmu_copytte)
908
909
910 /*
911 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
912 * %o0 = TSB base address (in), pointer to TSB entry (out)
913 * %o1 = vaddr (in)
914 * %o2 = vpshift (in)
915 * %o3 = tsb size code (in)
916 * %o4 = scratch register
917 */
918 ENTRY_NP(sfmmu_get_tsbe)
919 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
920 retl
921 nop
922 SET_SIZE(sfmmu_get_tsbe)
923
924 /*
925 * Return a TSB tag for the given va.
926 * %o0 = va (in/clobbered)
927 * %o0 = va shifted to be in tsb tag format (with no context) (out)
928 */
929 ENTRY_NP(sfmmu_make_tsbtag)
930 retl
931 srln %o0, TTARGET_VA_SHIFT, %o0
932 SET_SIZE(sfmmu_make_tsbtag)
933
934 #endif /* lint */
935
936 /*
937 * Other sfmmu primitives
938 */
939
940
941 #if defined (lint)
942 void
943 sfmmu_patch_ktsb(void)
944 {
945 }
946
947 void
948 sfmmu_kpm_patch_tlbm(void)
949 {
950 }
951
952 void
953 sfmmu_kpm_patch_tsbm(void)
954 {
955 }
956
957 void
958 sfmmu_patch_shctx(void)
959 {
960 }
961
962 /* ARGSUSED */
963 void
964 sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
965 {
966 }
967
968 /* ARGSUSED */
969 void
970 sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
971 {
972 }
973
974 /* ARGSUSED */
975 void
976 sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
977 {
978 }
979
980 /* ARGSUSED */
981 void
982 sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
983 {
984 }
985
986 #else /* lint */
987
988 #define I_SIZE 4
989
990 ENTRY_NP(sfmmu_fix_ktlb_traptable)
991 /*
992 * %o0 = start of patch area
993 * %o1 = size code of TSB to patch
994 * %o3 = scratch
995 */
996 /* fix sll */
997 ld [%o0], %o3 /* get sll */
998 sub %o3, %o1, %o3 /* decrease shift by tsb szc */
999 st %o3, [%o0] /* write sll */
1000 flush %o0
1001 /* fix srl */
1002 add %o0, I_SIZE, %o0 /* goto next instr. */
1003 ld [%o0], %o3 /* get srl */
1004 sub %o3, %o1, %o3 /* decrease shift by tsb szc */
1005 st %o3, [%o0] /* write srl */
1006 retl
1007 flush %o0
1008 SET_SIZE(sfmmu_fix_ktlb_traptable)
1009
1010 ENTRY_NP(sfmmu_fixup_ktsbbase)
1011 /*
1012 * %o0 = start of patch area
1013 * %o5 = kernel virtual or physical tsb base address
1014 * %o2, %o3 are used as scratch registers.
1015 */
1016 /* fixup sethi instruction */
1017 ld [%o0], %o3
1018 srl %o5, 10, %o2 ! offset is bits 32:10
1019 or %o3, %o2, %o3 ! set imm22
1020 st %o3, [%o0]
1021 /* fixup offset of lduw/ldx */
1022 add %o0, I_SIZE, %o0 ! next instr
1023 ld [%o0], %o3
1024 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0
1025 or %o3, %o2, %o3
1026 st %o3, [%o0]
1027 retl
1028 flush %o0
1029 SET_SIZE(sfmmu_fixup_ktsbbase)
1030
1031 ENTRY_NP(sfmmu_fixup_setx)
1032 /*
1033 * %o0 = start of patch area
1034 * %o4 = 64 bit value to patch
1035 * %o2, %o3 are used as scratch registers.
1036 *
1037 * Note: Assuming that all parts of the instructions which need to be
1038 * patched correspond to RUNTIME_PATCH (aka 0)
1039 *
1040 * Note the implementation of setx which is being patched is as follows:
1041 *
1042 * sethi %hh(RUNTIME_PATCH), tmp
1043 * sethi %lm(RUNTIME_PATCH), dest
1044 * or tmp, %hm(RUNTIME_PATCH), tmp
1045 * or dest, %lo(RUNTIME_PATCH), dest
1046 * sllx tmp, 32, tmp
1047 * nop
1048 * or tmp, dest, dest
1049 *
1050 * which differs from the implementation in the
1051 * "SPARC Architecture Manual"
1052 */
1053 /* fixup sethi instruction */
1054 ld [%o0], %o3
1055 srlx %o4, 42, %o2 ! bits [63:42]
1056 or %o3, %o2, %o3 ! set imm22
1057 st %o3, [%o0]
1058 /* fixup sethi instruction */
1059 add %o0, I_SIZE, %o0 ! next instr
1060 ld [%o0], %o3
1061 sllx %o4, 32, %o2 ! clear upper bits
1062 srlx %o2, 42, %o2 ! bits [31:10]
1063 or %o3, %o2, %o3 ! set imm22
1064 st %o3, [%o0]
1065 /* fixup or instruction */
1066 add %o0, I_SIZE, %o0 ! next instr
1067 ld [%o0], %o3
1068 srlx %o4, 32, %o2 ! bits [63:32]
1069 and %o2, 0x3ff, %o2 ! bits [41:32]
1070 or %o3, %o2, %o3 ! set imm
1071 st %o3, [%o0]
1072 /* fixup or instruction */
1073 add %o0, I_SIZE, %o0 ! next instr
1074 ld [%o0], %o3
1075 and %o4, 0x3ff, %o2 ! bits [9:0]
1076 or %o3, %o2, %o3 ! set imm
1077 st %o3, [%o0]
1078 retl
1079 flush %o0
1080 SET_SIZE(sfmmu_fixup_setx)
1081
1082 ENTRY_NP(sfmmu_fixup_or)
1083 /*
1084 * %o0 = start of patch area
1085 * %o4 = 32 bit value to patch
1086 * %o2, %o3 are used as scratch registers.
1087 * Note: Assuming that all parts of the instructions which need to be
1088 * patched correspond to RUNTIME_PATCH (aka 0)
1089 */
1090 ld [%o0], %o3
1091 and %o4, 0x3ff, %o2 ! bits [9:0]
1092 or %o3, %o2, %o3 ! set imm
1093 st %o3, [%o0]
1094 retl
1095 flush %o0
1096 SET_SIZE(sfmmu_fixup_or)
1097
1098 ENTRY_NP(sfmmu_fixup_shiftx)
1099 /*
1100 * %o0 = start of patch area
1101 * %o4 = signed int immediate value to add to sllx/srlx imm field
1102 * %o2, %o3 are used as scratch registers.
1103 *
1104 * sllx/srlx store the 6 bit immediate value in the lowest order bits
1105 * so we do a simple add. The caller must be careful to prevent
1106 * overflow, which could easily occur if the initial value is nonzero!
1107 */
1108 ld [%o0], %o3 ! %o3 = instruction to patch
1109 and %o3, 0x3f, %o2 ! %o2 = existing imm value
1110 add %o2, %o4, %o2 ! %o2 = new imm value
1111 andn %o3, 0x3f, %o3 ! clear old imm value
1112 and %o2, 0x3f, %o2 ! truncate new imm value
1113 or %o3, %o2, %o3 ! set new imm value
1114 st %o3, [%o0] ! store updated instruction
1115 retl
1116 flush %o0
1117 SET_SIZE(sfmmu_fixup_shiftx)
1118
1119 ENTRY_NP(sfmmu_fixup_mmu_asi)
1120 /*
1121 * Patch imm_asi of all ldda instructions in the MMU
1122 * trap handlers. We search MMU_PATCH_INSTR instructions
1123 * starting from the itlb miss handler (trap 0x64).
1124 * %o0 = address of tt[0,1]_itlbmiss
1125 * %o1 = imm_asi to setup, shifted by appropriate offset.
1126 * %o3 = number of instructions to search
1127 * %o4 = reserved by caller: called from leaf routine
1128 */
1129 1: ldsw [%o0], %o2 ! load instruction to %o2
1130 brgez,pt %o2, 2f
1131 srl %o2, 30, %o5
1132 btst 1, %o5 ! test bit 30; skip if not set
1133 bz,pt %icc, 2f
1134 sllx %o2, 39, %o5 ! bit 24 -> bit 63
1135 srlx %o5, 58, %o5 ! isolate op3 part of opcode
1136 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda
1137 brnz,pt %o5, 2f ! skip if not a match
1138 or %o2, %o1, %o2 ! or in imm_asi
1139 st %o2, [%o0] ! write patched instruction
1140 2: dec %o3
1141 brnz,a,pt %o3, 1b ! loop until we're done
1142 add %o0, I_SIZE, %o0
1143 retl
1144 flush %o0
1145 SET_SIZE(sfmmu_fixup_mmu_asi)
1146
1147 /*
1148 * Patch immediate ASI used to access the TSB in the
1149 * trap table.
1150 * inputs: %o0 = value of ktsb_phys
1151 */
1152 ENTRY_NP(sfmmu_patch_mmu_asi)
1153 mov %o7, %o4 ! save return pc in %o4
1154 mov ASI_QUAD_LDD_PHYS, %o3 ! set QUAD_LDD_PHYS by default
1155
1156 #ifdef sun4v
1157
1158 /*
1159 * Check ktsb_phys. It must be non-zero for sun4v, panic if not.
1160 */
1161
1162 brnz,pt %o0, do_patch
1163 nop
1164
1165 sethi %hi(sfmmu_panic11), %o0
1166 call panic
1167 or %o0, %lo(sfmmu_panic11), %o0
1168 do_patch:
1169
1170 #else /* sun4v */
1171 /*
1172 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0).
1173 * Note that ASI_NQUAD_LD is not defined/used for sun4v
1174 */
1175 movrz %o0, ASI_NQUAD_LD, %o3
1176
1177 #endif /* sun4v */
1178
1179 sll %o3, 5, %o1 ! imm_asi offset
1180 mov 6, %o3 ! number of instructions
1181 sethi %hi(dktsb), %o0 ! to search
1182 call sfmmu_fixup_mmu_asi ! patch kdtlb miss
1183 or %o0, %lo(dktsb), %o0
1184 mov 6, %o3 ! number of instructions
1185 sethi %hi(dktsb4m), %o0 ! to search
1186 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss
1187 or %o0, %lo(dktsb4m), %o0
1188 mov 6, %o3 ! number of instructions
1189 sethi %hi(iktsb), %o0 ! to search
1190 call sfmmu_fixup_mmu_asi ! patch kitlb miss
1191 or %o0, %lo(iktsb), %o0
1192 mov 6, %o3 ! number of instructions
1193 sethi %hi(iktsb4m), %o0 ! to search
1194 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss
1195 or %o0, %lo(iktsb4m), %o0
1196 mov %o4, %o7 ! retore return pc -- leaf
1197 retl
1198 nop
1199 SET_SIZE(sfmmu_patch_mmu_asi)
1200
1201
1202 ENTRY_NP(sfmmu_patch_ktsb)
1203 /*
1204 * We need to fix iktsb, dktsb, et. al.
1205 */
1206 save %sp, -SA(MINFRAME), %sp
1207 set ktsb_phys, %o1
1208 ld [%o1], %o4
1209 set ktsb_base, %o5
1210 set ktsb4m_base, %l1
1211 brz,pt %o4, 1f
1212 nop
1213 set ktsb_pbase, %o5
1214 set ktsb4m_pbase, %l1
1215 1:
1216 sethi %hi(ktsb_szcode), %o1
1217 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */
1218
1219 sethi %hi(iktsb), %o0
1220 call sfmmu_fix_ktlb_traptable
1221 or %o0, %lo(iktsb), %o0
1222
1223 sethi %hi(dktsb), %o0
1224 call sfmmu_fix_ktlb_traptable
1225 or %o0, %lo(dktsb), %o0
1226
1227 sethi %hi(ktsb4m_szcode), %o1
1228 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */
1229
1230 sethi %hi(iktsb4m), %o0
1231 call sfmmu_fix_ktlb_traptable
1232 or %o0, %lo(iktsb4m), %o0
1233
1234 sethi %hi(dktsb4m), %o0
1235 call sfmmu_fix_ktlb_traptable
1236 or %o0, %lo(dktsb4m), %o0
1237
1238 #ifndef sun4v
1239 mov ASI_N, %o2
1240 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch
1241 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4
1242 sethi %hi(tsb_kernel_patch_asi), %o0
1243 call sfmmu_fixup_or
1244 or %o0, %lo(tsb_kernel_patch_asi), %o0
1245 #endif /* !sun4v */
1246
1247 ldx [%o5], %o4 ! load ktsb base addr (VA or PA)
1248
1249 sethi %hi(dktsbbase), %o0
1250 call sfmmu_fixup_setx ! patch value of ktsb base addr
1251 or %o0, %lo(dktsbbase), %o0
1252
1253 sethi %hi(iktsbbase), %o0
1254 call sfmmu_fixup_setx ! patch value of ktsb base addr
1255 or %o0, %lo(iktsbbase), %o0
1256
1257 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0
1258 call sfmmu_fixup_setx ! patch value of ktsb base addr
1259 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1260
1261 #ifdef sun4v
1262 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0
1263 call sfmmu_fixup_setx ! patch value of ktsb base addr
1264 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1265 #endif /* sun4v */
1266
1267 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA)
1268
1269 sethi %hi(dktsb4mbase), %o0
1270 call sfmmu_fixup_setx ! patch value of ktsb4m base addr
1271 or %o0, %lo(dktsb4mbase), %o0
1272
1273 sethi %hi(iktsb4mbase), %o0
1274 call sfmmu_fixup_setx ! patch value of ktsb4m base addr
1275 or %o0, %lo(iktsb4mbase), %o0
1276
1277 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1278 call sfmmu_fixup_setx ! patch value of ktsb4m base addr
1279 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1280
1281 #ifdef sun4v
1282 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1283 call sfmmu_fixup_setx ! patch value of ktsb4m base addr
1284 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1285 #endif /* sun4v */
1286
1287 set ktsb_szcode, %o4
1288 ld [%o4], %o4
1289 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1290 call sfmmu_fixup_or ! patch value of ktsb_szcode
1291 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1292
1293 #ifdef sun4v
1294 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1295 call sfmmu_fixup_or ! patch value of ktsb_szcode
1296 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1297 #endif /* sun4v */
1298
1299 set ktsb4m_szcode, %o4
1300 ld [%o4], %o4
1301 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1302 call sfmmu_fixup_or ! patch value of ktsb4m_szcode
1303 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1304
1305 #ifdef sun4v
1306 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1307 call sfmmu_fixup_or ! patch value of ktsb4m_szcode
1308 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1309 #endif /* sun4v */
1310
1311 ret
1312 restore
1313 SET_SIZE(sfmmu_patch_ktsb)
1314
1315 ENTRY_NP(sfmmu_kpm_patch_tlbm)
1316 /*
1317 * Fixup trap handlers in common segkpm case. This is reserved
1318 * for future use should kpm TSB be changed to be other than the
1319 * kernel TSB.
1320 */
1321 retl
1322 nop
1323 SET_SIZE(sfmmu_kpm_patch_tlbm)
1324
1325 ENTRY_NP(sfmmu_kpm_patch_tsbm)
1326 /*
1327 * nop the branch to sfmmu_kpm_dtsb_miss_small
1328 * in the case where we are using large pages for
1329 * seg_kpm (and hence must probe the second TSB for
1330 * seg_kpm VAs)
1331 */
1332 set dktsb4m_kpmcheck_small, %o0
1333 MAKE_NOP_INSTR(%o1)
1334 st %o1, [%o0]
1335 flush %o0
1336 retl
1337 nop
1338 SET_SIZE(sfmmu_kpm_patch_tsbm)
1339
1340 ENTRY_NP(sfmmu_patch_utsb)
1341 #ifdef UTSB_PHYS
1342 retl
1343 nop
1344 #else /* UTSB_PHYS */
1345 /*
1346 * We need to hot patch utsb_vabase and utsb4m_vabase
1347 */
1348 save %sp, -SA(MINFRAME), %sp
1349
1350 /* patch value of utsb_vabase */
1351 set utsb_vabase, %o1
1352 ldx [%o1], %o4
1353 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1354 call sfmmu_fixup_setx
1355 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1356 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1357 call sfmmu_fixup_setx
1358 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1359 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1360 call sfmmu_fixup_setx
1361 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1362
1363 /* patch value of utsb4m_vabase */
1364 set utsb4m_vabase, %o1
1365 ldx [%o1], %o4
1366 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1367 call sfmmu_fixup_setx
1368 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1369 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1370 call sfmmu_fixup_setx
1371 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1372 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1373 call sfmmu_fixup_setx
1374 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1375
1376 /*
1377 * Patch TSB base register masks and shifts if needed.
1378 * By default the TSB base register contents are set up for 4M slab.
1379 * If we're using a smaller slab size and reserved VA range we need
1380 * to patch up those values here.
1381 */
1382 set tsb_slab_shift, %o1
1383 set MMU_PAGESHIFT4M, %o4
1384 lduw [%o1], %o3
1385 subcc %o4, %o3, %o4
1386 bz,pt %icc, 1f
1387 /* delay slot safe */
1388
1389 /* patch reserved VA range size if needed. */
1390 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0
1391 call sfmmu_fixup_shiftx
1392 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1393 call sfmmu_fixup_shiftx
1394 add %o0, I_SIZE, %o0
1395 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0
1396 call sfmmu_fixup_shiftx
1397 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1398 call sfmmu_fixup_shiftx
1399 add %o0, I_SIZE, %o0
1400 1:
1401 /* patch TSBREG_VAMASK used to set up TSB base register */
1402 set tsb_slab_mask, %o1
1403 ldx [%o1], %o4
1404 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1405 call sfmmu_fixup_or
1406 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1407 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1408 call sfmmu_fixup_or
1409 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1410
1411 ret
1412 restore
1413 #endif /* UTSB_PHYS */
1414 SET_SIZE(sfmmu_patch_utsb)
1415
1416 ENTRY_NP(sfmmu_patch_shctx)
1417 #ifdef sun4u
1418 retl
1419 nop
1420 #else /* sun4u */
1421 set sfmmu_shctx_cpu_mondo_patch, %o0
1422 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5
1423 st %o1, [%o0]
1424 flush %o0
1425 MAKE_NOP_INSTR(%o1)
1426 add %o0, I_SIZE, %o0 ! next instr
1427 st %o1, [%o0]
1428 flush %o0
1429
1430 set sfmmu_shctx_user_rtt_patch, %o0
1431 st %o1, [%o0] ! nop 1st instruction
1432 flush %o0
1433 add %o0, I_SIZE, %o0
1434 st %o1, [%o0] ! nop 2nd instruction
1435 flush %o0
1436 add %o0, I_SIZE, %o0
1437 st %o1, [%o0] ! nop 3rd instruction
1438 flush %o0
1439 add %o0, I_SIZE, %o0
1440 st %o1, [%o0] ! nop 4th instruction
1441 flush %o0
1442 add %o0, I_SIZE, %o0
1443 st %o1, [%o0] ! nop 5th instruction
1444 flush %o0
1445 add %o0, I_SIZE, %o0
1446 st %o1, [%o0] ! nop 6th instruction
1447 retl
1448 flush %o0
1449 #endif /* sun4u */
1450 SET_SIZE(sfmmu_patch_shctx)
1451
1452 /*
1453 * Routine that loads an entry into a tsb using virtual addresses.
1454 * Locking is required since all cpus can use the same TSB.
1455 * Note that it is no longer required to have a valid context
1456 * when calling this function.
1457 */
1458 ENTRY_NP(sfmmu_load_tsbe)
1459 /*
1460 * %o0 = pointer to tsbe to load
1461 * %o1 = tsb tag
1462 * %o2 = virtual pointer to TTE
1463 * %o3 = 1 if physical address in %o0 else 0
1464 */
1465 rdpr %pstate, %o5
1466 #ifdef DEBUG
1467 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1468 #endif /* DEBUG */
1469
1470 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */
1471
1472 SETUP_TSB_ASI(%o3, %g3)
1473 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8)
1474
1475 wrpr %g0, %o5, %pstate /* enable interrupts */
1476
1477 retl
1478 membar #StoreStore|#StoreLoad
1479 SET_SIZE(sfmmu_load_tsbe)
1480
1481 /*
1482 * Flush TSB of a given entry if the tag matches.
1483 */
1484 ENTRY(sfmmu_unload_tsbe)
1485 /*
1486 * %o0 = pointer to tsbe to be flushed
1487 * %o1 = tag to match
1488 * %o2 = 1 if physical address in %o0 else 0
1489 */
1490 SETUP_TSB_ASI(%o2, %g1)
1491 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1492 retl
1493 membar #StoreStore|#StoreLoad
1494 SET_SIZE(sfmmu_unload_tsbe)
1495
1496 /*
1497 * Routine that loads a TTE into the kpm TSB from C code.
1498 * Locking is required since kpm TSB is shared among all CPUs.
1499 */
1500 ENTRY_NP(sfmmu_kpm_load_tsb)
1501 /*
1502 * %o0 = vaddr
1503 * %o1 = ttep
1504 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1505 */
1506 rdpr %pstate, %o5 ! %o5 = saved pstate
1507 #ifdef DEBUG
1508 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1509 #endif /* DEBUG */
1510 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts
1511
1512 #ifndef sun4v
1513 sethi %hi(ktsb_phys), %o4
1514 mov ASI_N, %o3
1515 ld [%o4 + %lo(ktsb_phys)], %o4
1516 movrnz %o4, ASI_MEM, %o3
1517 mov %o3, %asi
1518 #endif /* !sun4v */
1519 mov %o0, %g1 ! %g1 = vaddr
1520
1521 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1522 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1523 /* %g2 = tsbep, %g1 clobbered */
1524
1525 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target
1526 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1527 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9)
1528
1529 wrpr %g0, %o5, %pstate ! enable interrupts
1530 retl
1531 membar #StoreStore|#StoreLoad
1532 SET_SIZE(sfmmu_kpm_load_tsb)
1533
1534 /*
1535 * Routine that shoots down a TTE in the kpm TSB or in the
1536 * kernel TSB depending on virtpg. Locking is required since
1537 * kpm/kernel TSB is shared among all CPUs.
1538 */
1539 ENTRY_NP(sfmmu_kpm_unload_tsb)
1540 /*
1541 * %o0 = vaddr
1542 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1543 */
1544 #ifndef sun4v
1545 sethi %hi(ktsb_phys), %o4
1546 mov ASI_N, %o3
1547 ld [%o4 + %lo(ktsb_phys)], %o4
1548 movrnz %o4, ASI_MEM, %o3
1549 mov %o3, %asi
1550 #endif /* !sun4v */
1551 mov %o0, %g1 ! %g1 = vaddr
1552
1553 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1554 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1555 /* %g2 = tsbep, %g1 clobbered */
1556
1557 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target
1558 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1559 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1560
1561 retl
1562 membar #StoreStore|#StoreLoad
1563 SET_SIZE(sfmmu_kpm_unload_tsb)
1564
1565 #endif /* lint */
1566
1567
1568 #if defined (lint)
1569
1570 /*ARGSUSED*/
1571 pfn_t
1572 sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1573 { return(0); }
1574
1575 #else /* lint */
1576
1577 ENTRY_NP(sfmmu_ttetopfn)
1578 ldx [%o0], %g1 /* read tte */
1579 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1580 /*
1581 * g1 = pfn
1582 */
1583 retl
1584 mov %g1, %o0
1585 SET_SIZE(sfmmu_ttetopfn)
1586
1587 #endif /* !lint */
1588
1589 /*
1590 * These macros are used to update global sfmmu hme hash statistics
1591 * in perf critical paths. It is only enabled in debug kernels or
1592 * if SFMMU_STAT_GATHER is defined
1593 */
1594 #if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1595 #define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \
1596 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\
1597 mov HATSTAT_KHASH_SEARCH, tmp2 ;\
1598 cmp tmp1, hatid ;\
1599 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\
1600 set sfmmu_global_stat, tmp1 ;\
1601 add tmp1, tmp2, tmp1 ;\
1602 ld [tmp1], tmp2 ;\
1603 inc tmp2 ;\
1604 st tmp2, [tmp1]
1605
1606 #define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \
1607 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\
1608 mov HATSTAT_KHASH_LINKS, tmp2 ;\
1609 cmp tmp1, hatid ;\
1610 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\
1611 set sfmmu_global_stat, tmp1 ;\
1612 add tmp1, tmp2, tmp1 ;\
1613 ld [tmp1], tmp2 ;\
1614 inc tmp2 ;\
1615 st tmp2, [tmp1]
1616
1617
1618 #else /* DEBUG || SFMMU_STAT_GATHER */
1619
1620 #define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1621
1622 #define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1623
1624 #endif /* DEBUG || SFMMU_STAT_GATHER */
1625
1626 /*
1627 * This macro is used to update global sfmmu kstas in non
1628 * perf critical areas so they are enabled all the time
1629 */
1630 #define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \
1631 sethi %hi(sfmmu_global_stat), tmp1 ;\
1632 add tmp1, statname, tmp1 ;\
1633 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\
1634 inc tmp2 ;\
1635 st tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1636
1637 /*
1638 * These macros are used to update per cpu stats in non perf
1639 * critical areas so they are enabled all the time
1640 */
1641 #define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \
1642 ld [tsbarea + stat], tmp1 ;\
1643 inc tmp1 ;\
1644 st tmp1, [tsbarea + stat]
1645
1646 /*
1647 * These macros are used to update per cpu stats in non perf
1648 * critical areas so they are enabled all the time
1649 */
1650 #define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \
1651 lduh [tsbarea + stat], tmp1 ;\
1652 inc tmp1 ;\
1653 stuh tmp1, [tsbarea + stat]
1654
1655 #if defined(KPM_TLBMISS_STATS_GATHER)
1656 /*
1657 * Count kpm dtlb misses separately to allow a different
1658 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1659 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1660 */
1661 #define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \
1662 brgez tagacc, label /* KPM VA? */ ;\
1663 nop ;\
1664 CPU_INDEX(tmp1, tsbma) ;\
1665 sethi %hi(kpmtsbm_area), tsbma ;\
1666 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\
1667 or tsbma, %lo(kpmtsbm_area), tsbma ;\
1668 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\
1669 /* VA range check */ ;\
1670 ldx [tsbma + KPMTSBM_VBASE], val ;\
1671 cmp tagacc, val ;\
1672 blu,pn %xcc, label ;\
1673 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\
1674 cmp tagacc, tmp1 ;\
1675 bgeu,pn %xcc, label ;\
1676 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\
1677 inc val ;\
1678 st val, [tsbma + KPMTSBM_DTLBMISS] ;\
1679 label:
1680 #else
1681 #define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1682 #endif /* KPM_TLBMISS_STATS_GATHER */
1683
1684 #if defined (lint)
1685 /*
1686 * The following routines are jumped to from the mmu trap handlers to do
1687 * the setting up to call systrap. They are separate routines instead of
1688 * being part of the handlers because the handlers would exceed 32
1689 * instructions and since this is part of the slow path the jump
1690 * cost is irrelevant.
1691 */
1692 void
1693 sfmmu_pagefault(void)
1694 {
1695 }
1696
1697 void
1698 sfmmu_mmu_trap(void)
1699 {
1700 }
1701
1702 void
1703 sfmmu_window_trap(void)
1704 {
1705 }
1706
1707 void
1708 sfmmu_kpm_exception(void)
1709 {
1710 }
1711
1712 #else /* lint */
1713
1714 #ifdef PTL1_PANIC_DEBUG
1715 .seg ".data"
1716 .global test_ptl1_panic
1717 test_ptl1_panic:
1718 .word 0
1719 .align 8
1720
1721 .seg ".text"
1722 .align 4
1723 #endif /* PTL1_PANIC_DEBUG */
1724
1725
1726 ENTRY_NP(sfmmu_pagefault)
1727 SET_GL_REG(1)
1728 USE_ALTERNATE_GLOBALS(%g5)
1729 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1730 rdpr %tt, %g6
1731 cmp %g6, FAST_IMMU_MISS_TT
1732 be,a,pn %icc, 1f
1733 mov T_INSTR_MMU_MISS, %g3
1734 cmp %g6, T_INSTR_MMU_MISS
1735 be,a,pn %icc, 1f
1736 mov T_INSTR_MMU_MISS, %g3
1737 mov %g5, %g2
1738 mov T_DATA_PROT, %g3 /* arg2 = traptype */
1739 cmp %g6, FAST_DMMU_MISS_TT
1740 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */
1741 cmp %g6, T_DATA_MMU_MISS
1742 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */
1743
1744 #ifdef PTL1_PANIC_DEBUG
1745 /* check if we want to test the tl1 panic */
1746 sethi %hi(test_ptl1_panic), %g4
1747 ld [%g4 + %lo(test_ptl1_panic)], %g1
1748 st %g0, [%g4 + %lo(test_ptl1_panic)]
1749 cmp %g1, %g0
1750 bne,a,pn %icc, ptl1_panic
1751 or %g0, PTL1_BAD_DEBUG, %g1
1752 #endif /* PTL1_PANIC_DEBUG */
1753 1:
1754 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1755 /*
1756 * g2 = tag access reg
1757 * g3.l = type
1758 * g3.h = 0
1759 */
1760 sethi %hi(trap), %g1
1761 or %g1, %lo(trap), %g1
1762 2:
1763 ba,pt %xcc, sys_trap
1764 mov -1, %g4
1765 SET_SIZE(sfmmu_pagefault)
1766
1767 ENTRY_NP(sfmmu_mmu_trap)
1768 SET_GL_REG(1)
1769 USE_ALTERNATE_GLOBALS(%g5)
1770 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1771 rdpr %tt, %g6
1772 cmp %g6, FAST_IMMU_MISS_TT
1773 be,a,pn %icc, 1f
1774 mov T_INSTR_MMU_MISS, %g3
1775 cmp %g6, T_INSTR_MMU_MISS
1776 be,a,pn %icc, 1f
1777 mov T_INSTR_MMU_MISS, %g3
1778 mov %g5, %g2
1779 mov T_DATA_PROT, %g3 /* arg2 = traptype */
1780 cmp %g6, FAST_DMMU_MISS_TT
1781 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */
1782 cmp %g6, T_DATA_MMU_MISS
1783 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */
1784 1:
1785 /*
1786 * g2 = tag access reg
1787 * g3 = type
1788 */
1789 sethi %hi(sfmmu_tsbmiss_exception), %g1
1790 or %g1, %lo(sfmmu_tsbmiss_exception), %g1
1791 ba,pt %xcc, sys_trap
1792 mov -1, %g4
1793 /*NOTREACHED*/
1794 SET_SIZE(sfmmu_mmu_trap)
1795
1796 ENTRY_NP(sfmmu_suspend_tl)
1797 SET_GL_REG(1)
1798 USE_ALTERNATE_GLOBALS(%g5)
1799 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1800 rdpr %tt, %g6
1801 cmp %g6, FAST_IMMU_MISS_TT
1802 be,a,pn %icc, 1f
1803 mov T_INSTR_MMU_MISS, %g3
1804 mov %g5, %g2
1805 cmp %g6, FAST_DMMU_MISS_TT
1806 move %icc, T_DATA_MMU_MISS, %g3
1807 movne %icc, T_DATA_PROT, %g3
1808 1:
1809 sethi %hi(sfmmu_tsbmiss_suspended), %g1
1810 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1
1811 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1812 ba,pt %xcc, sys_trap
1813 mov PIL_15, %g4
1814 /*NOTREACHED*/
1815 SET_SIZE(sfmmu_suspend_tl)
1816
1817 /*
1818 * No %g registers in use at this point.
1819 */
1820 ENTRY_NP(sfmmu_window_trap)
1821 rdpr %tpc, %g1
1822 #ifdef sun4v
1823 #ifdef DEBUG
1824 /* We assume previous %gl was 1 */
1825 rdpr %tstate, %g4
1826 srlx %g4, TSTATE_GL_SHIFT, %g4
1827 and %g4, TSTATE_GL_MASK, %g4
1828 cmp %g4, 1
1829 bne,a,pn %icc, ptl1_panic
1830 mov PTL1_BAD_WTRAP, %g1
1831 #endif /* DEBUG */
1832 /* user miss at tl>1. better be the window handler or user_rtt */
1833 /* in user_rtt? */
1834 set rtt_fill_start, %g4
1835 cmp %g1, %g4
1836 blu,pn %xcc, 6f
1837 .empty
1838 set rtt_fill_end, %g4
1839 cmp %g1, %g4
1840 bgeu,pn %xcc, 6f
1841 nop
1842 set fault_rtt_fn1, %g1
1843 wrpr %g0, %g1, %tnpc
1844 ba,a 7f
1845 6:
1846 ! must save this trap level before descending trap stack
1847 ! no need to save %tnpc, either overwritten or discarded
1848 ! already got it: rdpr %tpc, %g1
1849 rdpr %tstate, %g6
1850 rdpr %tt, %g7
1851 ! trap level saved, go get underlying trap type
1852 rdpr %tl, %g5
1853 sub %g5, 1, %g3
1854 wrpr %g3, %tl
1855 rdpr %tt, %g2
1856 wrpr %g5, %tl
1857 ! restore saved trap level
1858 wrpr %g1, %tpc
1859 wrpr %g6, %tstate
1860 wrpr %g7, %tt
1861 #else /* sun4v */
1862 /* user miss at tl>1. better be the window handler */
1863 rdpr %tl, %g5
1864 sub %g5, 1, %g3
1865 wrpr %g3, %tl
1866 rdpr %tt, %g2
1867 wrpr %g5, %tl
1868 #endif /* sun4v */
1869 and %g2, WTRAP_TTMASK, %g4
1870 cmp %g4, WTRAP_TYPE
1871 bne,pn %xcc, 1f
1872 nop
1873 /* tpc should be in the trap table */
1874 set trap_table, %g4
1875 cmp %g1, %g4
1876 blt,pn %xcc, 1f
1877 .empty
1878 set etrap_table, %g4
1879 cmp %g1, %g4
1880 bge,pn %xcc, 1f
1881 .empty
1882 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */
1883 add %g1, WTRAP_FAULTOFF, %g1
1884 wrpr %g0, %g1, %tnpc
1885 7:
1886 /*
1887 * some wbuf handlers will call systrap to resolve the fault
1888 * we pass the trap type so they figure out the correct parameters.
1889 * g5 = trap type, g6 = tag access reg
1890 */
1891
1892 /*
1893 * only use g5, g6, g7 registers after we have switched to alternate
1894 * globals.
1895 */
1896 SET_GL_REG(1)
1897 USE_ALTERNATE_GLOBALS(%g5)
1898 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1899 rdpr %tt, %g7
1900 cmp %g7, FAST_IMMU_MISS_TT
1901 be,a,pn %icc, ptl1_panic
1902 mov PTL1_BAD_WTRAP, %g1
1903 cmp %g7, T_INSTR_MMU_MISS
1904 be,a,pn %icc, ptl1_panic
1905 mov PTL1_BAD_WTRAP, %g1
1906 mov T_DATA_PROT, %g5
1907 cmp %g7, FAST_DMMU_MISS_TT
1908 move %icc, T_DATA_MMU_MISS, %g5
1909 cmp %g7, T_DATA_MMU_MISS
1910 move %icc, T_DATA_MMU_MISS, %g5
1911 ! XXXQ AGS re-check out this one
1912 done
1913 1:
1914 CPU_PADDR(%g1, %g4)
1915 add %g1, CPU_TL1_HDLR, %g1
1916 lda [%g1]ASI_MEM, %g4
1917 brnz,a,pt %g4, sfmmu_mmu_trap
1918 sta %g0, [%g1]ASI_MEM
1919 ba,pt %icc, ptl1_panic
1920 mov PTL1_BAD_TRAP, %g1
1921 SET_SIZE(sfmmu_window_trap)
1922
1923 ENTRY_NP(sfmmu_kpm_exception)
1924 /*
1925 * We have accessed an unmapped segkpm address or a legal segkpm
1926 * address which is involved in a VAC alias conflict prevention.
1927 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1928 * set. If it is, we will instead note that a fault has occurred
1929 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1930 * a "retry"). This will step over the faulting instruction.
1931 * Note that this means that a legal segkpm address involved in
1932 * a VAC alias conflict prevention (a rare case to begin with)
1933 * cannot be used in DTrace.
1934 */
1935 CPU_INDEX(%g1, %g2)
1936 set cpu_core, %g2
1937 sllx %g1, CPU_CORE_SHIFT, %g1
1938 add %g1, %g2, %g1
1939 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2
1940 andcc %g2, CPU_DTRACE_NOFAULT, %g0
1941 bz 0f
1942 or %g2, CPU_DTRACE_BADADDR, %g2
1943 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS]
1944 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1945 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL]
1946 done
1947 0:
1948 TSTAT_CHECK_TL1(1f, %g1, %g2)
1949 1:
1950 SET_GL_REG(1)
1951 USE_ALTERNATE_GLOBALS(%g5)
1952 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1953 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */
1954 /*
1955 * g2=tagacc g3.l=type g3.h=0
1956 */
1957 sethi %hi(trap), %g1
1958 or %g1, %lo(trap), %g1
1959 ba,pt %xcc, sys_trap
1960 mov -1, %g4
1961 SET_SIZE(sfmmu_kpm_exception)
1962
1963 #endif /* lint */
1964
1965 #if defined (lint)
1966
1967 void
1968 sfmmu_tsb_miss(void)
1969 {
1970 }
1971
1972 void
1973 sfmmu_kpm_dtsb_miss(void)
1974 {
1975 }
1976
1977 void
1978 sfmmu_kpm_dtsb_miss_small(void)
1979 {
1980 }
1981
1982 #else /* lint */
1983
1984 #if (IMAP_SEG != 0)
1985 #error - ism_map->ism_seg offset is not zero
1986 #endif
1987
1988 /*
1989 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1990 * tlb miss and branches to label "ismhit". If this is not an ISM
1991 * process or an ISM tlb miss it falls thru.
1992 *
1993 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1994 * this process.
1995 * If so, it will branch to label "ismhit". If not, it will fall through.
1996 *
1997 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1998 * so that any other threads of this process will not try and walk the ism
1999 * maps while they are being changed.
2000 *
2001 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
2002 * will make sure of that. This means we can terminate our search on
2003 * the first zero mapping we find.
2004 *
2005 * Parameters:
2006 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in)
2007 * tsbmiss = address of tsb miss area (in)
2008 * ismseg = contents of ism_seg for this ism map (out)
2009 * ismhat = physical address of imap_ismhat for this ism map (out)
2010 * tmp1 = scratch reg (CLOBBERED)
2011 * tmp2 = scratch reg (CLOBBERED)
2012 * tmp3 = scratch reg (CLOBBERED)
2013 * label: temporary labels
2014 * ismhit: label where to jump to if an ism dtlb miss
2015 * exitlabel:label where to jump if hat is busy due to hat_unshare.
2016 */
2017 #define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
2018 label, ismhit) \
2019 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\
2020 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\
2021 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\
2022 label/**/1: ;\
2023 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\
2024 mov tmp1, tmp3 /* update current ismblkpa head */ ;\
2025 label/**/2: ;\
2026 brz,pt ismseg, label/**/3 /* no mapping */ ;\
2027 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\
2028 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\
2029 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\
2030 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\
2031 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\
2032 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\
2033 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\
2034 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\
2035 cmp tmp2, tmp1 /* check va <= offset*/ ;\
2036 blu,a,pt %xcc, ismhit /* ism hit */ ;\
2037 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\
2038 ;\
2039 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\
2040 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\
2041 cmp ismhat, tmp1 ;\
2042 bl,pt %xcc, label/**/2 /* keep looking */ ;\
2043 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\
2044 ;\
2045 add tmp3, IBLK_NEXTPA, tmp1 ;\
2046 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\
2047 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\
2048 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\
2049 label/**/3:
2050
2051 /*
2052 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
2053 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
2054 * Parameters:
2055 * tagacc = reg containing virtual address
2056 * hatid = reg containing sfmmu pointer
2057 * hmeshift = constant/register to shift vaddr to obtain vapg
2058 * hmebp = register where bucket pointer will be stored
2059 * vapg = register where virtual page will be stored
2060 * tmp1, tmp2 = tmp registers
2061 */
2062
2063
2064 #define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \
2065 vapg, label, tmp1, tmp2) \
2066 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\
2067 brnz,a,pt tmp1, label/**/1 ;\
2068 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\
2069 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\
2070 ba,pt %xcc, label/**/2 ;\
2071 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\
2072 label/**/1: ;\
2073 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\
2074 label/**/2: ;\
2075 srlx tagacc, hmeshift, vapg ;\
2076 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\
2077 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\
2078 mulx hmebp, HMEBUCK_SIZE, hmebp ;\
2079 add hmebp, tmp1, hmebp
2080
2081 /*
2082 * hashtag includes bspage + hashno (64 bits).
2083 */
2084
2085 #define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \
2086 sllx vapg, hmeshift, vapg ;\
2087 mov hashno, hblktag ;\
2088 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\
2089 or vapg, hblktag, hblktag
2090
2091 /*
2092 * Function to traverse hmeblk hash link list and find corresponding match.
2093 * The search is done using physical pointers. It returns the physical address
2094 * pointer to the hmeblk that matches with the tag provided.
2095 * Parameters:
2096 * hmebp = register that points to hme hash bucket, also used as
2097 * tmp reg (clobbered)
2098 * hmeblktag = register with hmeblk tag match
2099 * hatid = register with hatid
2100 * hmeblkpa = register where physical ptr will be stored
2101 * tmp1 = tmp reg
2102 * label: temporary label
2103 */
2104
2105 #define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \
2106 tmp1, label) \
2107 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\
2108 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\
2109 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\
2110 label/**/1: ;\
2111 cmp hmeblkpa, HMEBLK_ENDPA ;\
2112 be,pn %xcc, label/**/2 ;\
2113 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\
2114 add hmeblkpa, HMEBLK_TAG, hmebp ;\
2115 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\
2116 add hmebp, CLONGSIZE, hmebp ;\
2117 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\
2118 xor tmp1, hmeblktag, tmp1 ;\
2119 xor hmebp, hatid, hmebp ;\
2120 or hmebp, tmp1, hmebp ;\
2121 brz,pn hmebp, label/**/2 /* branch on hit */ ;\
2122 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\
2123 ba,pt %xcc, label/**/1 ;\
2124 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\
2125 label/**/2:
2126
2127 /*
2128 * Function to traverse hmeblk hash link list and find corresponding match.
2129 * The search is done using physical pointers. It returns the physical address
2130 * pointer to the hmeblk that matches with the tag
2131 * provided.
2132 * Parameters:
2133 * hmeblktag = register with hmeblk tag match (rid field is 0)
2134 * hatid = register with hatid (pointer to SRD)
2135 * hmeblkpa = register where physical ptr will be stored
2136 * tmp1 = tmp reg
2137 * tmp2 = tmp reg
2138 * label: temporary label
2139 */
2140
2141 #define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \
2142 tmp1, tmp2, label) \
2143 label/**/1: ;\
2144 cmp hmeblkpa, HMEBLK_ENDPA ;\
2145 be,pn %xcc, label/**/4 ;\
2146 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\
2147 add hmeblkpa, HMEBLK_TAG, tmp2 ;\
2148 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\
2149 add tmp2, CLONGSIZE, tmp2 ;\
2150 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\
2151 xor tmp1, hmeblktag, tmp1 ;\
2152 xor tmp2, hatid, tmp2 ;\
2153 brz,pn tmp2, label/**/3 /* branch on hit */ ;\
2154 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\
2155 label/**/2: ;\
2156 ba,pt %xcc, label/**/1 ;\
2157 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\
2158 label/**/3: ;\
2159 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\
2160 bgeu,pt %xcc, label/**/2 ;\
2161 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\
2162 and tmp1, BT_ULMASK, tmp2 ;\
2163 srlx tmp1, BT_ULSHIFT, tmp1 ;\
2164 sllx tmp1, CLONGSHIFT, tmp1 ;\
2165 add tsbarea, tmp1, tmp1 ;\
2166 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\
2167 srlx tmp1, tmp2, tmp1 ;\
2168 btst 0x1, tmp1 ;\
2169 bz,pn %xcc, label/**/2 ;\
2170 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\
2171 label/**/4:
2172
2173 #if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2174 #error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2175 #endif
2176
2177 /*
2178 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2179 * the offset for the corresponding hment.
2180 * Parameters:
2181 * In:
2182 * vaddr = register with virtual address
2183 * hmeblkpa = physical pointer to hme_blk
2184 * Out:
2185 * hmentoff = register where hment offset will be stored
2186 * hmemisc = hblk_misc
2187 * Scratch:
2188 * tmp1
2189 */
2190 #define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2191 add hmeblkpa, HMEBLK_MISC, hmentoff ;\
2192 lda [hmentoff]ASI_MEM, hmemisc ;\
2193 andcc hmemisc, HBLK_SZMASK, %g0 ;\
2194 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\
2195 or %g0, HMEBLK_HME1, hmentoff ;\
2196 srl vaddr, MMU_PAGESHIFT, tmp1 ;\
2197 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\
2198 sllx tmp1, SFHME_SHIFT, tmp1 ;\
2199 add tmp1, HMEBLK_HME1, hmentoff ;\
2200 label1:
2201
2202 /*
2203 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2204 *
2205 * tagacc = (pseudo-)tag access register (in)
2206 * hatid = sfmmu pointer for TSB miss (in)
2207 * tte = tte for TLB miss if found, otherwise clobbered (out)
2208 * hmeblkpa = PA of hment if found, otherwise clobbered (out)
2209 * tsbarea = pointer to the tsbmiss area for this cpu. (in)
2210 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered
2211 * hmeshift = constant/register to shift VA to obtain the virtual pfn
2212 * for this page size.
2213 * hashno = constant/register hash number
2214 * tmp = temp value - clobbered
2215 * label = temporary label for branching within macro.
2216 * foundlabel = label to jump to when tte is found.
2217 * suspendlabel= label to jump to when tte is suspended.
2218 * exitlabel = label to jump to when tte is not found.
2219 *
2220 */
2221 #define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \
2222 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \
2223 ;\
2224 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\
2225 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\
2226 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \
2227 hmeblkpa, label/**/5, hmemisc, tmp) ;\
2228 ;\
2229 /* ;\
2230 * tagacc = tagacc ;\
2231 * hatid = hatid ;\
2232 * tsbarea = tsbarea ;\
2233 * tte = hmebp (hme bucket pointer) ;\
2234 * hmeblkpa = vapg (virtual page) ;\
2235 * hmemisc, tmp = scratch ;\
2236 */ ;\
2237 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\
2238 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\
2239 ;\
2240 /* ;\
2241 * tagacc = tagacc ;\
2242 * hatid = hatid ;\
2243 * tte = hmebp ;\
2244 * hmeblkpa = CLOBBERED ;\
2245 * hmemisc = htag_bspage+hashno+invalid_rid ;\
2246 * tmp = scratch ;\
2247 */ ;\
2248 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\
2249 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \
2250 tsbarea, tagacc, label/**/1) ;\
2251 /* ;\
2252 * tagacc = CLOBBERED ;\
2253 * tte = CLOBBERED ;\
2254 * hmeblkpa = hmeblkpa ;\
2255 * tmp = scratch ;\
2256 */ ;\
2257 cmp hmeblkpa, HMEBLK_ENDPA ;\
2258 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\
2259 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\
2260 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\
2261 nop ;\
2262 label/**/4: ;\
2263 /* ;\
2264 * We have found the hmeblk containing the hment. ;\
2265 * Now we calculate the corresponding tte. ;\
2266 * ;\
2267 * tagacc = tagacc ;\
2268 * hatid = hatid ;\
2269 * tte = clobbered ;\
2270 * hmeblkpa = hmeblkpa ;\
2271 * hmemisc = hblktag ;\
2272 * tmp = scratch ;\
2273 */ ;\
2274 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \
2275 label/**/2) ;\
2276 ;\
2277 /* ;\
2278 * tagacc = tagacc ;\
2279 * hatid = hmentoff ;\
2280 * tte = clobbered ;\
2281 * hmeblkpa = hmeblkpa ;\
2282 * hmemisc = hblk_misc ;\
2283 * tmp = scratch ;\
2284 */ ;\
2285 ;\
2286 add hatid, SFHME_TTE, hatid ;\
2287 add hmeblkpa, hatid, hmeblkpa ;\
2288 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\
2289 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\
2290 set TTE_SUSPEND, hatid ;\
2291 TTE_SUSPEND_INT_SHIFT(hatid) ;\
2292 btst tte, hatid ;\
2293 bz,pt %xcc, foundlabel ;\
2294 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\
2295 ;\
2296 /* ;\
2297 * Mapping is suspended, so goto suspend label. ;\
2298 */ ;\
2299 ba,pt %xcc, suspendlabel ;\
2300 nop
2301
2302 /*
2303 * GET_SHME_TTE is similar to GET_TTE() except it searches
2304 * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2305 * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2306 * either 0 (not part of scd) or 1 (part of scd).
2307 */
2308 #define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \
2309 hmeshift, hashno, tmp, label, foundlabel, \
2310 suspendlabel, exitlabel) \
2311 ;\
2312 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\
2313 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\
2314 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \
2315 hmeblkpa, label/**/5, hmemisc, tmp) ;\
2316 ;\
2317 /* ;\
2318 * tagacc = tagacc ;\
2319 * hatid = hatid ;\
2320 * tsbarea = tsbarea ;\
2321 * tte = hmebp (hme bucket pointer) ;\
2322 * hmeblkpa = vapg (virtual page) ;\
2323 * hmemisc, tmp = scratch ;\
2324 */ ;\
2325 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\
2326 ;\
2327 /* ;\
2328 * tagacc = tagacc ;\
2329 * hatid = hatid ;\
2330 * tsbarea = tsbarea ;\
2331 * tte = hmebp ;\
2332 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\
2333 * hmeblkpa = CLOBBERED ;\
2334 * tmp = scratch ;\
2335 */ ;\
2336 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\
2337 ;\
2338 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\
2339 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\
2340 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\
2341 ;\
2342 label/**/8: ;\
2343 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \
2344 tsbarea, tagacc, tte, label/**/1) ;\
2345 /* ;\
2346 * tagacc = CLOBBERED ;\
2347 * tte = CLOBBERED ;\
2348 * hmeblkpa = hmeblkpa ;\
2349 * tmp = scratch ;\
2350 */ ;\
2351 cmp hmeblkpa, HMEBLK_ENDPA ;\
2352 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\
2353 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\
2354 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\
2355 nop ;\
2356 label/**/4: ;\
2357 /* ;\
2358 * We have found the hmeblk containing the hment. ;\
2359 * Now we calculate the corresponding tte. ;\
2360 * ;\
2361 * tagacc = tagacc ;\
2362 * hatid = hatid ;\
2363 * tte = clobbered ;\
2364 * hmeblkpa = hmeblkpa ;\
2365 * hmemisc = hblktag ;\
2366 * tsbarea = tsbmiss area ;\
2367 * tmp = scratch ;\
2368 */ ;\
2369 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \
2370 label/**/2) ;\
2371 ;\
2372 /* ;\
2373 * tagacc = tagacc ;\
2374 * hatid = hmentoff ;\
2375 * tte = clobbered ;\
2376 * hmeblkpa = hmeblkpa ;\
2377 * hmemisc = hblk_misc ;\
2378 * tsbarea = tsbmiss area ;\
2379 * tmp = scratch ;\
2380 */ ;\
2381 ;\
2382 add hatid, SFHME_TTE, hatid ;\
2383 add hmeblkpa, hatid, hmeblkpa ;\
2384 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\
2385 brlz,pt tte, label/**/6 ;\
2386 nop ;\
2387 btst HBLK_SZMASK, hmemisc ;\
2388 bnz,a,pt %icc, label/**/7 ;\
2389 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\
2390 ;\
2391 /* ;\
2392 * We found an invalid 8K tte in shme. ;\
2393 * it may not belong to shme's region since ;\
2394 * region size/alignment granularity is 8K but different ;\
2395 * regions don't share hmeblks. Continue the search. ;\
2396 */ ;\
2397 sub hmeblkpa, hatid, hmeblkpa ;\
2398 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\
2399 srlx tagacc, hmeshift, tte ;\
2400 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\
2401 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\
2402 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\
2403 ba,a,pt %xcc, label/**/8 ;\
2404 label/**/6: ;\
2405 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\
2406 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\
2407 label/**/7: ;\
2408 set TTE_SUSPEND, hatid ;\
2409 TTE_SUSPEND_INT_SHIFT(hatid) ;\
2410 btst tte, hatid ;\
2411 bz,pt %xcc, foundlabel ;\
2412 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\
2413 ;\
2414 /* ;\
2415 * Mapping is suspended, so goto suspend label. ;\
2416 */ ;\
2417 ba,pt %xcc, suspendlabel ;\
2418 nop
2419
2420 /*
2421 * KERNEL PROTECTION HANDLER
2422 *
2423 * g1 = tsb8k pointer register (clobbered)
2424 * g2 = tag access register (ro)
2425 * g3 - g7 = scratch registers
2426 *
2427 * Note: This function is patched at runtime for performance reasons.
2428 * Any changes here require sfmmu_patch_ktsb fixed.
2429 */
2430 ENTRY_NP(sfmmu_kprot_trap)
2431 mov %g2, %g7 ! TSB pointer macro clobbers tagacc
2432 sfmmu_kprot_patch_ktsb_base:
2433 RUNTIME_PATCH_SETX(%g1, %g6)
2434 /* %g1 = contents of ktsb_base or ktsb_pbase */
2435 sfmmu_kprot_patch_ktsb_szcode:
2436 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched)
2437
2438 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2439 ! %g1 = First TSB entry pointer, as TSB miss handler expects
2440
2441 mov %g2, %g7 ! TSB pointer macro clobbers tagacc
2442 sfmmu_kprot_patch_ktsb4m_base:
2443 RUNTIME_PATCH_SETX(%g3, %g6)
2444 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2445 sfmmu_kprot_patch_ktsb4m_szcode:
2446 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched)
2447
2448 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2449 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2450
2451 CPU_TSBMISS_AREA(%g6, %g7)
2452 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2453 ba,pt %xcc, sfmmu_tsb_miss_tt
2454 nop
2455
2456 /*
2457 * USER PROTECTION HANDLER
2458 *
2459 * g1 = tsb8k pointer register (ro)
2460 * g2 = tag access register (ro)
2461 * g3 = faulting context (clobbered, currently not used)
2462 * g4 - g7 = scratch registers
2463 */
2464 ALTENTRY(sfmmu_uprot_trap)
2465 #ifdef sun4v
2466 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2467 /* %g1 = first TSB entry ptr now, %g2 preserved */
2468
2469 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */
2470 brlz,pt %g3, 9f /* check for 2nd TSB */
2471 nop
2472
2473 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2474 /* %g3 = second TSB entry ptr now, %g2 preserved */
2475
2476 #else /* sun4v */
2477 #ifdef UTSB_PHYS
2478 /* g1 = first TSB entry ptr */
2479 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2480 brlz,pt %g3, 9f /* check for 2nd TSB */
2481 nop
2482
2483 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2484 /* %g3 = second TSB entry ptr now, %g2 preserved */
2485 #else /* UTSB_PHYS */
2486 brgez,pt %g1, 9f /* check for 2nd TSB */
2487 mov -1, %g3 /* set second tsbe ptr to -1 */
2488
2489 mov %g2, %g7
2490 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2491 /* %g3 = second TSB entry ptr now, %g7 clobbered */
2492 mov %g1, %g7
2493 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2494 #endif /* UTSB_PHYS */
2495 #endif /* sun4v */
2496 9:
2497 CPU_TSBMISS_AREA(%g6, %g7)
2498 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2499 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */
2500 nop
2501
2502 /*
2503 * Kernel 8K page iTLB miss. We also get here if we took a
2504 * fast instruction access mmu miss trap while running in
2505 * invalid context.
2506 *
2507 * %g1 = 8K TSB pointer register (not used, clobbered)
2508 * %g2 = tag access register (used)
2509 * %g3 = faulting context id (used)
2510 * %g7 = TSB tag to match (used)
2511 */
2512 .align 64
2513 ALTENTRY(sfmmu_kitlb_miss)
2514 brnz,pn %g3, tsb_tl0_noctxt
2515 nop
2516
2517 /* kernel miss */
2518 /* get kernel tsb pointer */
2519 /* we patch the next set of instructions at run time */
2520 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2521 iktsbbase:
2522 RUNTIME_PATCH_SETX(%g4, %g5)
2523 /* %g4 = contents of ktsb_base or ktsb_pbase */
2524
2525 iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2526 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2527 or %g4, %g1, %g1 ! form tsb ptr
2528 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data
2529 cmp %g4, %g7
2530 bne,pn %xcc, iktsb4mbase ! check 4m ktsb
2531 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index
2532
2533 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit
2534 bz,pn %icc, exec_fault
2535 nop
2536 TT_TRACE(trace_tsbhit) ! 2 instr traptrace
2537 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2538 retry
2539
2540 iktsb4mbase:
2541 RUNTIME_PATCH_SETX(%g4, %g6)
2542 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2543 iktsb4m:
2544 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2545 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2546 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr
2547 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data
2548 cmp %g4, %g7
2549 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss
2550 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit
2551 bz,pn %icc, exec_fault
2552 nop
2553 TT_TRACE(trace_tsbhit) ! 2 instr traptrace
2554 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2555 retry
2556
2557 /*
2558 * Kernel dTLB miss. We also get here if we took a fast data
2559 * access mmu miss trap while running in invalid context.
2560 *
2561 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2562 * We select the TSB miss handler to branch to depending on
2563 * the virtual address of the access. In the future it may
2564 * be desirable to separate kpm TTEs into their own TSB,
2565 * in which case all that needs to be done is to set
2566 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2567 * early in the miss if we detect a kpm VA to a new handler.
2568 *
2569 * %g1 = 8K TSB pointer register (not used, clobbered)
2570 * %g2 = tag access register (used)
2571 * %g3 = faulting context id (used)
2572 */
2573 .align 64
2574 ALTENTRY(sfmmu_kdtlb_miss)
2575 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */
2576 nop
2577
2578 /* Gather some stats for kpm misses in the TLB. */
2579 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2580 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2581
2582 /*
2583 * Get first TSB offset and look for 8K/64K/512K mapping
2584 * using the 8K virtual page as the index.
2585 *
2586 * We patch the next set of instructions at run time;
2587 * any changes here require sfmmu_patch_ktsb changes too.
2588 */
2589 dktsbbase:
2590 RUNTIME_PATCH_SETX(%g7, %g6)
2591 /* %g7 = contents of ktsb_base or ktsb_pbase */
2592
2593 dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2594 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2595
2596 /*
2597 * At this point %g1 is our index into the TSB.
2598 * We just masked off enough bits of the VA depending
2599 * on our TSB size code.
2600 */
2601 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data
2602 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare
2603 cmp %g6, %g4 ! compare tag
2604 bne,pn %xcc, dktsb4m_kpmcheck_small
2605 add %g7, %g1, %g1 /* form tsb ptr */
2606 TT_TRACE(trace_tsbhit)
2607 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2608 /* trapstat expects tte in %g5 */
2609 retry
2610
2611 /*
2612 * If kpm is using large pages, the following instruction needs
2613 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2614 * so that we will probe the 4M TSB regardless of the VA. In
2615 * the case kpm is using small pages, we know no large kernel
2616 * mappings are located above 0x80000000.00000000 so we skip the
2617 * probe as an optimization.
2618 */
2619 dktsb4m_kpmcheck_small:
2620 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2621 /* delay slot safe, below */
2622
2623 /*
2624 * Get second TSB offset and look for 4M mapping
2625 * using 4M virtual page as the TSB index.
2626 *
2627 * Here:
2628 * %g1 = 8K TSB pointer. Don't squash it.
2629 * %g2 = tag access register (we still need it)
2630 */
2631 srlx %g2, MMU_PAGESHIFT4M, %g3
2632
2633 /*
2634 * We patch the next set of instructions at run time;
2635 * any changes here require sfmmu_patch_ktsb changes too.
2636 */
2637 dktsb4mbase:
2638 RUNTIME_PATCH_SETX(%g7, %g6)
2639 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2640 dktsb4m:
2641 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2642 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2643
2644 /*
2645 * At this point %g3 is our index into the TSB.
2646 * We just masked off enough bits of the VA depending
2647 * on our TSB size code.
2648 */
2649 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data
2650 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare
2651 cmp %g6, %g4 ! compare tag
2652
2653 dktsb4m_tsbmiss:
2654 bne,pn %xcc, dktsb4m_kpmcheck
2655 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr
2656 TT_TRACE(trace_tsbhit)
2657 /* we don't check TTE size here since we assume 4M TSB is separate */
2658 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2659 /* trapstat expects tte in %g5 */
2660 retry
2661
2662 /*
2663 * So, we failed to find a valid TTE to match the faulting
2664 * address in either TSB. There are a few cases that could land
2665 * us here:
2666 *
2667 * 1) This is a kernel VA below 0x80000000.00000000. We branch
2668 * to sfmmu_tsb_miss_tt to handle the miss.
2669 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2670 * 4M TSB. Let segkpm handle it.
2671 *
2672 * Note that we shouldn't land here in the case of a kpm VA when
2673 * kpm_smallpages is active -- we handled that case earlier at
2674 * dktsb4m_kpmcheck_small.
2675 *
2676 * At this point:
2677 * g1 = 8K-indexed primary TSB pointer
2678 * g2 = tag access register
2679 * g3 = 4M-indexed secondary TSB pointer
2680 */
2681 dktsb4m_kpmcheck:
2682 cmp %g2, %g0
2683 bl,pn %xcc, sfmmu_kpm_dtsb_miss
2684 nop
2685 ba,a,pt %icc, sfmmu_tsb_miss_tt
2686 nop
2687
2688 #ifdef sun4v
2689 /*
2690 * User instruction miss w/ single TSB.
2691 * The first probe covers 8K, 64K, and 512K page sizes,
2692 * because 64K and 512K mappings are replicated off 8K
2693 * pointer.
2694 *
2695 * g1 = tsb8k pointer register
2696 * g2 = tag access register
2697 * g3 - g6 = scratch registers
2698 * g7 = TSB tag to match
2699 */
2700 .align 64
2701 ALTENTRY(sfmmu_uitlb_fastpath)
2702
2703 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2704 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2705 ba,pn %xcc, sfmmu_tsb_miss_tt
2706 mov -1, %g3
2707
2708 /*
2709 * User data miss w/ single TSB.
2710 * The first probe covers 8K, 64K, and 512K page sizes,
2711 * because 64K and 512K mappings are replicated off 8K
2712 * pointer.
2713 *
2714 * g1 = tsb8k pointer register
2715 * g2 = tag access register
2716 * g3 - g6 = scratch registers
2717 * g7 = TSB tag to match
2718 */
2719 .align 64
2720 ALTENTRY(sfmmu_udtlb_fastpath)
2721
2722 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2723 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2724 ba,pn %xcc, sfmmu_tsb_miss_tt
2725 mov -1, %g3
2726
2727 /*
2728 * User instruction miss w/ multiple TSBs (sun4v).
2729 * The first probe covers 8K, 64K, and 512K page sizes,
2730 * because 64K and 512K mappings are replicated off 8K
2731 * pointer. Second probe covers 4M page size only.
2732 *
2733 * Just like sfmmu_udtlb_slowpath, except:
2734 * o Uses ASI_ITLB_IN
2735 * o checks for execute permission
2736 * o No ISM prediction.
2737 *
2738 * g1 = tsb8k pointer register
2739 * g2 = tag access register
2740 * g3 - g6 = scratch registers
2741 * g7 = TSB tag to match
2742 */
2743 .align 64
2744 ALTENTRY(sfmmu_uitlb_slowpath)
2745
2746 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2747 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2748 /* g4 - g5 = clobbered here */
2749
2750 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2751 /* g1 = first TSB pointer, g3 = second TSB pointer */
2752 srlx %g2, TAG_VALO_SHIFT, %g7
2753 PROBE_2ND_ITSB(%g3, %g7)
2754 /* NOT REACHED */
2755
2756 #else /* sun4v */
2757
2758 /*
2759 * User instruction miss w/ multiple TSBs (sun4u).
2760 * The first probe covers 8K, 64K, and 512K page sizes,
2761 * because 64K and 512K mappings are replicated off 8K
2762 * pointer. Probe of 1st TSB has already been done prior to entry
2763 * into this routine. For the UTSB_PHYS case we probe up to 3
2764 * valid other TSBs in the following order:
2765 * 1) shared TSB for 4M-256M pages
2766 * 2) private TSB for 4M-256M pages
2767 * 3) shared TSB for 8K-512K pages
2768 *
2769 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs
2770 * 4M-256M pages.
2771 *
2772 * Just like sfmmu_udtlb_slowpath, except:
2773 * o Uses ASI_ITLB_IN
2774 * o checks for execute permission
2775 * o No ISM prediction.
2776 *
2777 * g1 = tsb8k pointer register
2778 * g2 = tag access register
2779 * g4 - g6 = scratch registers
2780 * g7 = TSB tag to match
2781 */
2782 .align 64
2783 ALTENTRY(sfmmu_uitlb_slowpath)
2784
2785 #ifdef UTSB_PHYS
2786
2787 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2788 brlz,pt %g6, 1f
2789 nop
2790 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2791 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail)
2792 1:
2793 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2794 brlz,pt %g3, 2f
2795 nop
2796 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2797 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail)
2798 2:
2799 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2800 brlz,pt %g6, sfmmu_tsb_miss_tt
2801 nop
2802 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2803 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail)
2804 ba,pn %xcc, sfmmu_tsb_miss_tt
2805 nop
2806
2807 #else /* UTSB_PHYS */
2808 mov %g1, %g3 /* save tsb8k reg in %g3 */
2809 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2810 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2811 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */
2812 mov %g3, %g7 /* copy tsb8k reg in %g7 */
2813 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2814 /* g1 = first TSB pointer, g3 = second TSB pointer */
2815 srlx %g2, TAG_VALO_SHIFT, %g7
2816 PROBE_2ND_ITSB(%g3, %g7, isynth)
2817 ba,pn %xcc, sfmmu_tsb_miss_tt
2818 nop
2819
2820 #endif /* UTSB_PHYS */
2821 #endif /* sun4v */
2822
2823 #if defined(sun4u) && defined(UTSB_PHYS)
2824
2825 /*
2826 * We come here for ism predict DTLB_MISS case or if
2827 * if probe in first TSB failed.
2828 */
2829
2830 .align 64
2831 ALTENTRY(sfmmu_udtlb_slowpath_noismpred)
2832
2833 /*
2834 * g1 = tsb8k pointer register
2835 * g2 = tag access register
2836 * g4 - %g6 = scratch registers
2837 * g7 = TSB tag to match
2838 */
2839
2840 /*
2841 * ISM non-predict probe order
2842 * probe 1ST_TSB (8K index)
2843 * probe 2ND_TSB (4M index)
2844 * probe 4TH_TSB (4M index)
2845 * probe 3RD_TSB (8K index)
2846 *
2847 * We already probed first TSB in DTLB_MISS handler.
2848 */
2849
2850 /*
2851 * Private 2ND TSB 4M-256 pages
2852 */
2853 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2854 brlz,pt %g3, 1f
2855 nop
2856 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2857 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2858
2859 /*
2860 * Shared Context 4TH TSB 4M-256 pages
2861 */
2862 1:
2863 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2864 brlz,pt %g6, 2f
2865 nop
2866 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2867 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail)
2868
2869 /*
2870 * Shared Context 3RD TSB 8K-512K pages
2871 */
2872 2:
2873 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2874 brlz,pt %g6, sfmmu_tsb_miss_tt
2875 nop
2876 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2877 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail)
2878 ba,pn %xcc, sfmmu_tsb_miss_tt
2879 nop
2880
2881 .align 64
2882 ALTENTRY(sfmmu_udtlb_slowpath_ismpred)
2883
2884 /*
2885 * g1 = tsb8k pointer register
2886 * g2 = tag access register
2887 * g4 - g6 = scratch registers
2888 * g7 = TSB tag to match
2889 */
2890
2891 /*
2892 * ISM predict probe order
2893 * probe 4TH_TSB (4M index)
2894 * probe 2ND_TSB (4M index)
2895 * probe 1ST_TSB (8K index)
2896 * probe 3RD_TSB (8K index)
2897
2898 /*
2899 * Shared Context 4TH TSB 4M-256 pages
2900 */
2901 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2902 brlz,pt %g6, 4f
2903 nop
2904 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2905 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2)
2906
2907 /*
2908 * Private 2ND TSB 4M-256 pages
2909 */
2910 4:
2911 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2912 brlz,pt %g3, 5f
2913 nop
2914 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2915 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2)
2916
2917 5:
2918 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2)
2919
2920 /*
2921 * Shared Context 3RD TSB 8K-512K pages
2922 */
2923 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2924 brlz,pt %g6, 6f
2925 nop
2926 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2927 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2)
2928 6:
2929 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */
2930 nop
2931
2932 #else /* sun4u && UTSB_PHYS */
2933
2934 .align 64
2935 ALTENTRY(sfmmu_udtlb_slowpath)
2936
2937 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */
2938 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */
2939 mov %g1, %g3
2940
2941 udtlb_miss_probefirst:
2942 /*
2943 * g1 = 8K TSB pointer register
2944 * g2 = tag access register
2945 * g3 = (potentially) second TSB entry ptr
2946 * g6 = ism pred.
2947 * g7 = vpg_4m
2948 */
2949 #ifdef sun4v
2950 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2951 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2952
2953 /*
2954 * Here:
2955 * g1 = first TSB pointer
2956 * g2 = tag access reg
2957 * g3 = second TSB ptr IFF ISM pred. (else don't care)
2958 */
2959 brgz,pn %g6, sfmmu_tsb_miss_tt
2960 nop
2961 #else /* sun4v */
2962 mov %g1, %g4
2963 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2964 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2965
2966 /*
2967 * Here:
2968 * g1 = first TSB pointer
2969 * g2 = tag access reg
2970 * g3 = second TSB ptr IFF ISM pred. (else don't care)
2971 */
2972 brgz,pn %g6, sfmmu_tsb_miss_tt
2973 nop
2974 ldxa [%g0]ASI_DMMU_TSB_8K, %g3
2975 /* fall through in 8K->4M probe order */
2976 #endif /* sun4v */
2977
2978 udtlb_miss_probesecond:
2979 /*
2980 * Look in the second TSB for the TTE
2981 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2982 * g2 = tag access reg
2983 * g3 = 8K TSB pointer register
2984 * g6 = ism pred.
2985 * g7 = vpg_4m
2986 */
2987 #ifdef sun4v
2988 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2989 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2990 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2991 #else /* sun4v */
2992 mov %g3, %g7
2993 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2994 /* %g2 clobbered, %g3 =second tsbe ptr */
2995 mov MMU_TAG_ACCESS, %g2
2996 ldxa [%g2]ASI_DMMU, %g2
2997 #endif /* sun4v */
2998
2999 srlx %g2, TAG_VALO_SHIFT, %g7
3000 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3001 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
3002 brgz,pn %g6, udtlb_miss_probefirst
3003 nop
3004
3005 /* fall through to sfmmu_tsb_miss_tt */
3006 #endif /* sun4u && UTSB_PHYS */
3007
3008
3009 ALTENTRY(sfmmu_tsb_miss_tt)
3010 TT_TRACE(trace_tsbmiss)
3011 /*
3012 * We get here if there is a TSB miss OR a write protect trap.
3013 *
3014 * g1 = First TSB entry pointer
3015 * g2 = tag access register
3016 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
3017 * g4 - g7 = scratch registers
3018 */
3019
3020 ALTENTRY(sfmmu_tsb_miss)
3021
3022 /*
3023 * If trapstat is running, we need to shift the %tpc and %tnpc to
3024 * point to trapstat's TSB miss return code (note that trapstat
3025 * itself will patch the correct offset to add).
3026 */
3027 rdpr %tl, %g7
3028 cmp %g7, 1
3029 ble,pt %xcc, 0f
3030 sethi %hi(KERNELBASE), %g6
3031 rdpr %tpc, %g7
3032 or %g6, %lo(KERNELBASE), %g6
3033 cmp %g7, %g6
3034 bgeu,pt %xcc, 0f
3035 /* delay slot safe */
3036
3037 ALTENTRY(tsbmiss_trapstat_patch_point)
3038 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */
3039 wrpr %g7, %tpc
3040 add %g7, 4, %g7
3041 wrpr %g7, %tnpc
3042 0:
3043 CPU_TSBMISS_AREA(%g6, %g7)
3044 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */
3045 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */
3046
3047 sllx %g2, TAGACC_CTX_LSHIFT, %g3
3048 brz,a,pn %g3, 1f /* skip ahead if kernel */
3049 ldn [%g6 + TSBMISS_KHATID], %g7
3050 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */
3051 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */
3052
3053 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
3054
3055 cmp %g3, INVALID_CONTEXT
3056 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */
3057 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
3058
3059 #if defined(sun4v) || defined(UTSB_PHYS)
3060 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */
3061 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */
3062 stub %g7, [%g6 + TSBMISS_URTTEFLAGS]
3063 #endif /* sun4v || UTSB_PHYS */
3064
3065 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
3066 /*
3067 * The miss wasn't in an ISM segment.
3068 *
3069 * %g1 %g3, %g4, %g5, %g7 all clobbered
3070 * %g2 = (pseudo) tag access
3071 */
3072
3073 ba,pt %icc, 2f
3074 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
3075
3076 1:
3077 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
3078 /*
3079 * 8K and 64K hash.
3080 */
3081 2:
3082
3083 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3084 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte,
3085 sfmmu_suspend_tl, tsb_512K)
3086 /* NOT REACHED */
3087
3088 tsb_512K:
3089 sllx %g2, TAGACC_CTX_LSHIFT, %g5
3090 brz,pn %g5, 3f
3091 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4
3092 and %g4, HAT_512K_FLAG, %g5
3093
3094 /*
3095 * Note that there is a small window here where we may have
3096 * a 512k page in the hash list but have not set the HAT_512K_FLAG
3097 * flag yet, so we will skip searching the 512k hash list.
3098 * In this case we will end up in pagefault which will find
3099 * the mapping and return. So, in this instance we will end up
3100 * spending a bit more time resolving this TSB miss, but it can
3101 * only happen once per process and even then, the chances of that
3102 * are very small, so it's not worth the extra overhead it would
3103 * take to close this window.
3104 */
3105 brz,pn %g5, tsb_4M
3106 nop
3107 3:
3108 /*
3109 * 512K hash
3110 */
3111
3112 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3113 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte,
3114 sfmmu_suspend_tl, tsb_4M)
3115 /* NOT REACHED */
3116
3117 tsb_4M:
3118 sllx %g2, TAGACC_CTX_LSHIFT, %g5
3119 brz,pn %g5, 4f
3120 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4
3121 and %g4, HAT_4M_FLAG, %g5
3122 brz,pn %g5, tsb_32M
3123 nop
3124 4:
3125 /*
3126 * 4M hash
3127 */
3128
3129 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3130 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte,
3131 sfmmu_suspend_tl, tsb_32M)
3132 /* NOT REACHED */
3133
3134 tsb_32M:
3135 sllx %g2, TAGACC_CTX_LSHIFT, %g5
3136 #ifdef sun4v
3137 brz,pn %g5, 6f
3138 #else
3139 brz,pn %g5, tsb_pagefault
3140 #endif
3141 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4
3142 and %g4, HAT_32M_FLAG, %g5
3143 brz,pn %g5, tsb_256M
3144 nop
3145 5:
3146 /*
3147 * 32M hash
3148 */
3149
3150 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3151 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte,
3152 sfmmu_suspend_tl, tsb_256M)
3153 /* NOT REACHED */
3154
3155 #if defined(sun4u) && !defined(UTSB_PHYS)
3156 #define tsb_shme tsb_pagefault
3157 #endif
3158 tsb_256M:
3159 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4
3160 and %g4, HAT_256M_FLAG, %g5
3161 brz,pn %g5, tsb_shme
3162 nop
3163 6:
3164 /*
3165 * 256M hash
3166 */
3167
3168 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3169 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte,
3170 sfmmu_suspend_tl, tsb_shme)
3171 /* NOT REACHED */
3172
3173 tsb_checktte:
3174 /*
3175 * g1 = hblk_misc
3176 * g2 = tagacc
3177 * g3 = tte
3178 * g4 = tte pa
3179 * g6 = tsbmiss area
3180 * g7 = hatid
3181 */
3182 brlz,a,pt %g3, tsb_validtte
3183 rdpr %tt, %g7
3184
3185 #if defined(sun4u) && !defined(UTSB_PHYS)
3186 #undef tsb_shme
3187 ba tsb_pagefault
3188 nop
3189 #else /* sun4u && !UTSB_PHYS */
3190
3191 tsb_shme:
3192 /*
3193 * g2 = tagacc
3194 * g6 = tsbmiss area
3195 */
3196 sllx %g2, TAGACC_CTX_LSHIFT, %g5
3197 brz,pn %g5, tsb_pagefault
3198 nop
3199 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */
3200 brz,pn %g7, tsb_pagefault
3201 nop
3202
3203 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3204 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte,
3205 sfmmu_suspend_tl, tsb_shme_512K)
3206 /* NOT REACHED */
3207
3208 tsb_shme_512K:
3209 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4
3210 and %g4, HAT_512K_FLAG, %g5
3211 brz,pn %g5, tsb_shme_4M
3212 nop
3213
3214 /*
3215 * 512K hash
3216 */
3217
3218 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3219 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte,
3220 sfmmu_suspend_tl, tsb_shme_4M)
3221 /* NOT REACHED */
3222
3223 tsb_shme_4M:
3224 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4
3225 and %g4, HAT_4M_FLAG, %g5
3226 brz,pn %g5, tsb_shme_32M
3227 nop
3228 4:
3229 /*
3230 * 4M hash
3231 */
3232 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3233 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte,
3234 sfmmu_suspend_tl, tsb_shme_32M)
3235 /* NOT REACHED */
3236
3237 tsb_shme_32M:
3238 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4
3239 and %g4, HAT_32M_FLAG, %g5
3240 brz,pn %g5, tsb_shme_256M
3241 nop
3242
3243 /*
3244 * 32M hash
3245 */
3246
3247 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3248 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte,
3249 sfmmu_suspend_tl, tsb_shme_256M)
3250 /* NOT REACHED */
3251
3252 tsb_shme_256M:
3253 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4
3254 and %g4, HAT_256M_FLAG, %g5
3255 brz,pn %g5, tsb_pagefault
3256 nop
3257
3258 /*
3259 * 256M hash
3260 */
3261
3262 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3263 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte,
3264 sfmmu_suspend_tl, tsb_pagefault)
3265 /* NOT REACHED */
3266
3267 tsb_shme_checktte:
3268
3269 brgez,pn %g3, tsb_pagefault
3270 rdpr %tt, %g7
3271 /*
3272 * g1 = ctx1 flag
3273 * g3 = tte
3274 * g4 = tte pa
3275 * g6 = tsbmiss area
3276 * g7 = tt
3277 */
3278
3279 brz,pt %g1, tsb_validtte
3280 nop
3281 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1
3282 or %g1, HAT_CHKCTX1_FLAG, %g1
3283 stub %g1, [%g6 + TSBMISS_URTTEFLAGS]
3284
3285 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3286 #endif /* sun4u && !UTSB_PHYS */
3287
3288 tsb_validtte:
3289 /*
3290 * g3 = tte
3291 * g4 = tte pa
3292 * g6 = tsbmiss area
3293 * g7 = tt
3294 */
3295
3296 /*
3297 * Set ref/mod bits if this is a prot trap. Usually, it isn't.
3298 */
3299 cmp %g7, FAST_PROT_TT
3300 bne,pt %icc, 4f
3301 nop
3302
3303 TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod,
3304 tsb_protfault)
3305
3306 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */
3307 #ifdef sun4v
3308 MMU_FAULT_STATUS_AREA(%g7)
3309 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */
3310 #else /* sun4v */
3311 mov MMU_TAG_ACCESS, %g5
3312 ldxa [%g5]ASI_DMMU, %g5
3313 #endif /* sun4v */
3314 ba,pt %xcc, tsb_update_tl1
3315 nop
3316 4:
3317 /*
3318 * If ITLB miss check exec bit.
3319 * If not set treat as invalid TTE.
3320 */
3321 cmp %g7, T_INSTR_MMU_MISS
3322 be,pn %icc, 5f
3323 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */
3324 cmp %g7, FAST_IMMU_MISS_TT
3325 bne,pt %icc, 3f
3326 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */
3327 5:
3328 bz,pn %icc, tsb_protfault
3329 nop
3330
3331 3:
3332 /*
3333 * Set reference bit if not already set
3334 */
3335 TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref)
3336
3337 /*
3338 * Now, load into TSB/TLB. At this point:
3339 * g3 = tte
3340 * g4 = patte
3341 * g6 = tsbmiss area
3342 */
3343 rdpr %tt, %g7
3344 #ifdef sun4v
3345 MMU_FAULT_STATUS_AREA(%g2)
3346 cmp %g7, T_INSTR_MMU_MISS
3347 be,a,pt %icc, 9f
3348 nop
3349 cmp %g7, FAST_IMMU_MISS_TT
3350 be,a,pt %icc, 9f
3351 nop
3352 add %g2, MMFSA_D_, %g2
3353 9:
3354 ldx [%g2 + MMFSA_CTX_], %g7
3355 sllx %g7, TTARGET_CTX_SHIFT, %g7
3356 ldx [%g2 + MMFSA_ADDR_], %g2
3357 mov %g2, %g5 ! load the fault addr for later use
3358 srlx %g2, TTARGET_VA_SHIFT, %g2
3359 or %g2, %g7, %g2
3360 #else /* sun4v */
3361 mov MMU_TAG_ACCESS, %g5
3362 cmp %g7, FAST_IMMU_MISS_TT
3363 be,a,pt %icc, 9f
3364 ldxa [%g0]ASI_IMMU, %g2
3365 ldxa [%g0]ASI_DMMU, %g2
3366 ba,pt %icc, tsb_update_tl1
3367 ldxa [%g5]ASI_DMMU, %g5
3368 9:
3369 ldxa [%g5]ASI_IMMU, %g5
3370 #endif /* sun4v */
3371
3372 tsb_update_tl1:
3373 srlx %g2, TTARGET_CTX_SHIFT, %g7
3374 brz,pn %g7, tsb_kernel
3375 #ifdef sun4v
3376 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0
3377 #else /* sun4v */
3378 srlx %g3, TTE_SZ_SHFT, %g7
3379 #endif /* sun4v */
3380
3381 tsb_user:
3382 #ifdef sun4v
3383 cmp %g7, TTE4M
3384 bge,pn %icc, tsb_user4m
3385 nop
3386 #else /* sun4v */
3387 cmp %g7, TTESZ_VALID | TTE4M
3388 be,pn %icc, tsb_user4m
3389 srlx %g3, TTE_SZ2_SHFT, %g7
3390 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB
3391 #ifdef ITLB_32M_256M_SUPPORT
3392 bnz,pn %icc, tsb_user4m
3393 nop
3394 #else /* ITLB_32M_256M_SUPPORT */
3395 bnz,a,pn %icc, tsb_user_pn_synth
3396 nop
3397 #endif /* ITLB_32M_256M_SUPPORT */
3398 #endif /* sun4v */
3399
3400 tsb_user8k:
3401 #if defined(sun4v) || defined(UTSB_PHYS)
3402 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7
3403 and %g7, HAT_CHKCTX1_FLAG, %g1
3404 brz,a,pn %g1, 1f
3405 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr
3406 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3407 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb
3408 mov PTL1_NO_SCDTSB8K, %g1 ! panic
3409 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
3410 1:
3411 #else /* defined(sun4v) || defined(UTSB_PHYS) */
3412 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr
3413 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
3414
3415 #ifndef UTSB_PHYS
3416 mov ASI_N, %g7 ! user TSBs accessed by VA
3417 mov %g7, %asi
3418 #endif /* !UTSB_PHYS */
3419
3420 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3)
3421
3422 rdpr %tt, %g5
3423 #ifdef sun4v
3424 cmp %g5, T_INSTR_MMU_MISS
3425 be,a,pn %xcc, 9f
3426 mov %g3, %g5
3427 #endif /* sun4v */
3428 cmp %g5, FAST_IMMU_MISS_TT
3429 be,pn %xcc, 9f
3430 mov %g3, %g5
3431
3432 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3433 ! trapstat wants TTE in %g5
3434 retry
3435 9:
3436 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3437 ! trapstat wants TTE in %g5
3438 retry
3439
3440 tsb_user4m:
3441 #if defined(sun4v) || defined(UTSB_PHYS)
3442 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7
3443 and %g7, HAT_CHKCTX1_FLAG, %g1
3444 brz,a,pn %g1, 4f
3445 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr
3446 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr
3447 brlz,a,pn %g1, 5f ! if no shared 4TH TSB
3448 nop
3449 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3450
3451 #else /* defined(sun4v) || defined(UTSB_PHYS) */
3452 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr
3453 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
3454 4:
3455 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */
3456 nop
3457
3458 #ifndef UTSB_PHYS
3459 mov ASI_N, %g7 ! user TSBs accessed by VA
3460 mov %g7, %asi
3461 #endif /* UTSB_PHYS */
3462
3463 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4)
3464
3465 5:
3466 rdpr %tt, %g5
3467 #ifdef sun4v
3468 cmp %g5, T_INSTR_MMU_MISS
3469 be,a,pn %xcc, 9f
3470 mov %g3, %g5
3471 #endif /* sun4v */
3472 cmp %g5, FAST_IMMU_MISS_TT
3473 be,pn %xcc, 9f
3474 mov %g3, %g5
3475
3476 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3477 ! trapstat wants TTE in %g5
3478 retry
3479 9:
3480 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3481 ! trapstat wants TTE in %g5
3482 retry
3483
3484 #if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3485 /*
3486 * Panther ITLB synthesis.
3487 * The Panther 32M and 256M ITLB code simulates these two large page
3488 * sizes with 4M pages, to provide support for programs, for example
3489 * Java, that may copy instructions into a 32M or 256M data page and
3490 * then execute them. The code below generates the 4M pfn bits and
3491 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3492 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3493 * are ignored by the hardware.
3494 *
3495 * Now, load into TSB/TLB. At this point:
3496 * g2 = tagtarget
3497 * g3 = tte
3498 * g4 = patte
3499 * g5 = tt
3500 * g6 = tsbmiss area
3501 */
3502 tsb_user_pn_synth:
3503 rdpr %tt, %g5
3504 cmp %g5, FAST_IMMU_MISS_TT
3505 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */
3506 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */
3507 bz,pn %icc, 4b /* if not, been here before */
3508 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */
3509 brlz,a,pn %g1, 5f /* no 2nd tsb */
3510 mov %g3, %g5
3511
3512 mov MMU_TAG_ACCESS, %g7
3513 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */
3514 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */
3515
3516 mov ASI_N, %g7 /* user TSBs always accessed by VA */
3517 mov %g7, %asi
3518 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */
3519 5:
3520 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3521 retry
3522
3523 tsb_user_itlb_synth:
3524 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */
3525
3526 mov MMU_TAG_ACCESS, %g7
3527 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */
3528 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */
3529 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */
3530 or %g5, %g3, %g5 /* add 4M bits to TTE */
3531
3532 mov ASI_N, %g7 /* user TSBs always accessed by VA */
3533 mov %g7, %asi
3534 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */
3535 7:
3536 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */
3537 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3538 retry
3539 #endif /* sun4v && ITLB_32M_256M_SUPPORT */
3540
3541 tsb_kernel:
3542 rdpr %tt, %g5
3543 #ifdef sun4v
3544 cmp %g7, TTE4M
3545 bge,pn %icc, 5f
3546 #else
3547 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support
3548 be,pn %icc, 5f
3549 #endif /* sun4v */
3550 nop
3551 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr
3552 ba,pt %xcc, 6f
3553 nop
3554 5:
3555 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr
3556 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */
3557 nop
3558 6:
3559 #ifndef sun4v
3560 tsb_kernel_patch_asi:
3561 or %g0, RUNTIME_PATCH, %g6
3562 mov %g6, %asi ! XXX avoid writing to %asi !!
3563 #endif
3564 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7)
3565 3:
3566 #ifdef sun4v
3567 cmp %g5, T_INSTR_MMU_MISS
3568 be,a,pn %icc, 1f
3569 mov %g3, %g5 ! trapstat wants TTE in %g5
3570 #endif /* sun4v */
3571 cmp %g5, FAST_IMMU_MISS_TT
3572 be,pn %icc, 1f
3573 mov %g3, %g5 ! trapstat wants TTE in %g5
3574 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3575 ! trapstat wants TTE in %g5
3576 retry
3577 1:
3578 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3579 ! trapstat wants TTE in %g5
3580 retry
3581
3582 tsb_ism:
3583 /*
3584 * This is an ISM [i|d]tlb miss. We optimize for largest
3585 * page size down to smallest.
3586 *
3587 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3588 * register
3589 * g3 = ismmap->ism_seg
3590 * g4 = physical address of ismmap->ism_sfmmu
3591 * g6 = tsbmiss area
3592 */
3593 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */
3594 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */
3595 mov PTL1_BAD_ISM, %g1
3596 /* g5 = pa of imap_vb_shift */
3597 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3598 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */
3599 srlx %g3, %g4, %g3 /* clr size field */
3600 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */
3601 sllx %g3, %g4, %g3 /* g3 = ism vbase */
3602 and %g2, %g1, %g4 /* g4 = ctx number */
3603 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */
3604 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */
3605 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */
3606 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3607 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */
3608 #if defined(sun4v) || defined(UTSB_PHYS)
3609 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */
3610 brz,pt %g5, tsb_chk4M_ism
3611 nop
3612 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5
3613 or %g5, HAT_CHKCTX1_FLAG, %g5
3614 stub %g5, [%g6 + TSBMISS_URTTEFLAGS]
3615 rdpr %tt, %g5
3616 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3617 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
3618
3619 /*
3620 * ISM pages are always locked down.
3621 * If we can't find the tte then pagefault
3622 * and let the spt segment driver resolve it.
3623 *
3624 * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3625 * g4 = imap_hatflags
3626 * g6 = tsb miss area
3627 * g7 = ISM hatid
3628 */
3629
3630 tsb_chk4M_ism:
3631 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */
3632 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */
3633 nop
3634
3635 tsb_ism_32M:
3636 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */
3637 brz,pn %g5, tsb_ism_256M
3638 nop
3639
3640 /*
3641 * 32M hash.
3642 */
3643
3644 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M,
3645 TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3646 tsb_ism_4M)
3647 /* NOT REACHED */
3648
3649 tsb_ism_32M_found:
3650 brlz,a,pt %g3, tsb_validtte
3651 rdpr %tt, %g7
3652 ba,pt %xcc, tsb_ism_4M
3653 nop
3654
3655 tsb_ism_256M:
3656 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */
3657 brz,a,pn %g5, ptl1_panic
3658 mov PTL1_BAD_ISM, %g1
3659
3660 /*
3661 * 256M hash.
3662 */
3663 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M,
3664 TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3665 tsb_ism_4M)
3666
3667 tsb_ism_256M_found:
3668 brlz,a,pt %g3, tsb_validtte
3669 rdpr %tt, %g7
3670
3671 tsb_ism_4M:
3672 /*
3673 * 4M hash.
3674 */
3675 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M,
3676 TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3677 tsb_ism_8K)
3678 /* NOT REACHED */
3679
3680 tsb_ism_4M_found:
3681 brlz,a,pt %g3, tsb_validtte
3682 rdpr %tt, %g7
3683
3684 tsb_ism_8K:
3685 /*
3686 * 8K and 64K hash.
3687 */
3688
3689 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K,
3690 TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3691 tsb_pagefault)
3692 /* NOT REACHED */
3693
3694 tsb_ism_8K_found:
3695 brlz,a,pt %g3, tsb_validtte
3696 rdpr %tt, %g7
3697
3698 tsb_pagefault:
3699 rdpr %tt, %g7
3700 cmp %g7, FAST_PROT_TT
3701 be,a,pn %icc, tsb_protfault
3702 wrpr %g0, FAST_DMMU_MISS_TT, %tt
3703
3704 tsb_protfault:
3705 /*
3706 * we get here if we couldn't find a valid tte in the hash.
3707 *
3708 * If user and we are at tl>1 we go to window handling code.
3709 *
3710 * If kernel and the fault is on the same page as our stack
3711 * pointer, then we know the stack is bad and the trap handler
3712 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3713 *
3714 * If this is a kernel trap and tl>1, panic.
3715 *
3716 * Otherwise we call pagefault.
3717 */
3718 cmp %g7, FAST_IMMU_MISS_TT
3719 #ifdef sun4v
3720 MMU_FAULT_STATUS_AREA(%g4)
3721 ldx [%g4 + MMFSA_I_CTX], %g5
3722 ldx [%g4 + MMFSA_D_CTX], %g4
3723 move %icc, %g5, %g4
3724 cmp %g7, T_INSTR_MMU_MISS
3725 move %icc, %g5, %g4
3726 #else
3727 mov MMU_TAG_ACCESS, %g4
3728 ldxa [%g4]ASI_DMMU, %g2
3729 ldxa [%g4]ASI_IMMU, %g5
3730 move %icc, %g5, %g2
3731 cmp %g7, T_INSTR_MMU_MISS
3732 move %icc, %g5, %g2
3733 sllx %g2, TAGACC_CTX_LSHIFT, %g4
3734 #endif /* sun4v */
3735 brnz,pn %g4, 3f /* skip if not kernel */
3736 rdpr %tl, %g5
3737
3738 add %sp, STACK_BIAS, %g3
3739 srlx %g3, MMU_PAGESHIFT, %g3
3740 srlx %g2, MMU_PAGESHIFT, %g4
3741 cmp %g3, %g4
3742 be,a,pn %icc, ptl1_panic /* panic if bad %sp */
3743 mov PTL1_BAD_STACK, %g1
3744
3745 cmp %g5, 1
3746 ble,pt %icc, 2f
3747 nop
3748 TSTAT_CHECK_TL1(2f, %g1, %g2)
3749 rdpr %tt, %g2
3750 cmp %g2, FAST_PROT_TT
3751 mov PTL1_BAD_KPROT_FAULT, %g1
3752 movne %icc, PTL1_BAD_KMISS, %g1
3753 ba,pt %icc, ptl1_panic
3754 nop
3755
3756 2:
3757 /*
3758 * We are taking a pagefault in the kernel on a kernel address. If
3759 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3760 * want to call sfmmu_pagefault -- we will instead note that a fault
3761 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3762 * (instead of a "retry"). This will step over the faulting
3763 * instruction.
3764 */
3765 CPU_INDEX(%g1, %g2)
3766 set cpu_core, %g2
3767 sllx %g1, CPU_CORE_SHIFT, %g1
3768 add %g1, %g2, %g1
3769 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2
3770 andcc %g2, CPU_DTRACE_NOFAULT, %g0
3771 bz sfmmu_pagefault
3772 or %g2, CPU_DTRACE_BADADDR, %g2
3773 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS]
3774 GET_MMU_D_ADDR(%g3, %g4)
3775 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL]
3776 done
3777
3778 3:
3779 cmp %g5, 1
3780 ble,pt %icc, 4f
3781 nop
3782 TSTAT_CHECK_TL1(4f, %g1, %g2)
3783 ba,pt %icc, sfmmu_window_trap
3784 nop
3785
3786 4:
3787 /*
3788 * We are taking a pagefault on a non-kernel address. If we are in
3789 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3790 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3791 */
3792 CPU_INDEX(%g1, %g2)
3793 set cpu_core, %g2
3794 sllx %g1, CPU_CORE_SHIFT, %g1
3795 add %g1, %g2, %g1
3796 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2
3797 andcc %g2, CPU_DTRACE_NOFAULT, %g0
3798 bz sfmmu_mmu_trap
3799 or %g2, CPU_DTRACE_BADADDR, %g2
3800 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS]
3801 GET_MMU_D_ADDR(%g3, %g4)
3802 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL]
3803
3804 /*
3805 * Be sure that we're actually taking this miss from the kernel --
3806 * otherwise we have managed to return to user-level with
3807 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3808 */
3809 rdpr %tstate, %g2
3810 btst TSTATE_PRIV, %g2
3811 bz,a ptl1_panic
3812 mov PTL1_BAD_DTRACE_FLAGS, %g1
3813 done
3814
3815 ALTENTRY(tsb_tl0_noctxt)
3816 /*
3817 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3818 * if it is, indicated that we have faulted and issue a done.
3819 */
3820 CPU_INDEX(%g5, %g6)
3821 set cpu_core, %g6
3822 sllx %g5, CPU_CORE_SHIFT, %g5
3823 add %g5, %g6, %g5
3824 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6
3825 andcc %g6, CPU_DTRACE_NOFAULT, %g0
3826 bz 1f
3827 or %g6, CPU_DTRACE_BADADDR, %g6
3828 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS]
3829 GET_MMU_D_ADDR(%g3, %g4)
3830 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL]
3831
3832 /*
3833 * Be sure that we're actually taking this miss from the kernel --
3834 * otherwise we have managed to return to user-level with
3835 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3836 */
3837 rdpr %tstate, %g5
3838 btst TSTATE_PRIV, %g5
3839 bz,a ptl1_panic
3840 mov PTL1_BAD_DTRACE_FLAGS, %g1
3841 TSTAT_CHECK_TL1(2f, %g1, %g2);
3842 2:
3843 done
3844
3845 1:
3846 rdpr %tt, %g5
3847 cmp %g5, FAST_IMMU_MISS_TT
3848 #ifdef sun4v
3849 MMU_FAULT_STATUS_AREA(%g2)
3850 be,a,pt %icc, 2f
3851 ldx [%g2 + MMFSA_I_CTX], %g3
3852 cmp %g5, T_INSTR_MMU_MISS
3853 be,a,pt %icc, 2f
3854 ldx [%g2 + MMFSA_I_CTX], %g3
3855 ldx [%g2 + MMFSA_D_CTX], %g3
3856 2:
3857 #else
3858 mov MMU_TAG_ACCESS, %g2
3859 be,a,pt %icc, 2f
3860 ldxa [%g2]ASI_IMMU, %g3
3861 ldxa [%g2]ASI_DMMU, %g3
3862 2: sllx %g3, TAGACC_CTX_LSHIFT, %g3
3863 #endif /* sun4v */
3864 brz,a,pn %g3, ptl1_panic ! panic if called for kernel
3865 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen
3866 rdpr %tl, %g5
3867 cmp %g5, 1
3868 ble,pt %icc, sfmmu_mmu_trap
3869 nop
3870 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3871 ba,pt %icc, sfmmu_window_trap
3872 nop
3873 SET_SIZE(sfmmu_tsb_miss)
3874 #endif /* lint */
3875
3876 #if defined (lint)
3877 /*
3878 * This routine will look for a user or kernel vaddr in the hash
3879 * structure. It returns a valid pfn or PFN_INVALID. It doesn't
3880 * grab any locks. It should only be used by other sfmmu routines.
3881 */
3882 /* ARGSUSED */
3883 pfn_t
3884 sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3885 {
3886 return(0);
3887 }
3888
3889 /* ARGSUSED */
3890 pfn_t
3891 sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno)
3892 {
3893 return(0);
3894 }
3895
3896 #else /* lint */
3897
3898 ENTRY_NP(sfmmu_vatopfn)
3899 /*
3900 * disable interrupts
3901 */
3902 rdpr %pstate, %o3
3903 #ifdef DEBUG
3904 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
3905 #endif
3906 /*
3907 * disable interrupts to protect the TSBMISS area
3908 */
3909 andn %o3, PSTATE_IE, %o5
3910 wrpr %o5, 0, %pstate
3911
3912 /*
3913 * o0 = vaddr
3914 * o1 = sfmmup
3915 * o2 = ttep
3916 */
3917 CPU_TSBMISS_AREA(%g1, %o5)
3918 ldn [%g1 + TSBMISS_KHATID], %o4
3919 cmp %o4, %o1
3920 bne,pn %ncc, vatopfn_nokernel
3921 mov TTE64K, %g5 /* g5 = rehash # */
3922 mov %g1,%o5 /* o5 = tsbmiss_area */
3923 /*
3924 * o0 = vaddr
3925 * o1 & o4 = hatid
3926 * o2 = ttep
3927 * o5 = tsbmiss area
3928 */
3929 mov HBLK_RANGE_SHIFT, %g6
3930 1:
3931
3932 /*
3933 * o0 = vaddr
3934 * o1 = sfmmup
3935 * o2 = ttep
3936 * o3 = old %pstate
3937 * o4 = hatid
3938 * o5 = tsbmiss
3939 * g5 = rehash #
3940 * g6 = hmeshift
3941 *
3942 * The first arg to GET_TTE is actually tagaccess register
3943 * not just vaddr. Since this call is for kernel we need to clear
3944 * any lower vaddr bits that would be interpreted as ctx bits.
3945 */
3946 set TAGACC_CTX_MASK, %g1
3947 andn %o0, %g1, %o0
3948 GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3,
3949 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3950
3951 kvtop_hblk_found:
3952 /*
3953 * o0 = vaddr
3954 * o1 = sfmmup
3955 * o2 = ttep
3956 * g1 = tte
3957 * g2 = tte pa
3958 * g3 = scratch
3959 * o2 = tsbmiss area
3960 * o1 = hat id
3961 */
3962 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */
3963 mov -1, %o0 /* output = -1 (PFN_INVALID) */
3964 stx %g1,[%o2] /* put tte into *ttep */
3965 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3966 /*
3967 * o0 = vaddr
3968 * o1 = sfmmup
3969 * o2 = ttep
3970 * g1 = pfn
3971 */
3972 ba,pt %xcc, 6f
3973 mov %g1, %o0
3974
3975 kvtop_nohblk:
3976 /*
3977 * we get here if we couldn't find valid hblk in hash. We rehash
3978 * if neccesary.
3979 */
3980 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3981 #ifdef sun4v
3982 cmp %g5, MAX_HASHCNT
3983 #else
3984 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */
3985 #endif /* sun4v */
3986 be,a,pn %icc, 6f
3987 mov -1, %o0 /* output = -1 (PFN_INVALID) */
3988 mov %o1, %o4 /* restore hatid */
3989 #ifdef sun4v
3990 add %g5, 2, %g5
3991 cmp %g5, 3
3992 move %icc, MMU_PAGESHIFT4M, %g6
3993 ba,pt %icc, 1b
3994 movne %icc, MMU_PAGESHIFT256M, %g6
3995 #else
3996 inc %g5
3997 cmp %g5, 2
3998 move %icc, MMU_PAGESHIFT512K, %g6
3999 ba,pt %icc, 1b
4000 movne %icc, MMU_PAGESHIFT4M, %g6
4001 #endif /* sun4v */
4002 6:
4003 retl
4004 wrpr %g0, %o3, %pstate /* re-enable interrupts */
4005
4006 tsb_suspend:
4007 /*
4008 * o0 = vaddr
4009 * o1 = sfmmup
4010 * o2 = ttep
4011 * g1 = tte
4012 * g2 = tte pa
4013 * g3 = tte va
4014 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss
4015 */
4016 stx %g1,[%o2] /* put tte into *ttep */
4017 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */
4018 sub %g0, 1, %o0 /* output = PFN_INVALID */
4019 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */
4020 8:
4021 retl
4022 wrpr %g0, %o3, %pstate /* enable interrupts */
4023
4024 vatopfn_nokernel:
4025 /*
4026 * This routine does NOT support user addresses
4027 * There is a routine in C that supports this.
4028 * The only reason why we don't have the C routine
4029 * support kernel addresses as well is because
4030 * we do va_to_pa while holding the hashlock.
4031 */
4032 wrpr %g0, %o3, %pstate /* re-enable interrupts */
4033 save %sp, -SA(MINFRAME), %sp
4034 sethi %hi(sfmmu_panic3), %o0
4035 call panic
4036 or %o0, %lo(sfmmu_panic3), %o0
4037
4038 SET_SIZE(sfmmu_vatopfn)
4039
4040 /*
4041 * %o0 = vaddr
4042 * %o1 = hashno (aka szc)
4043 *
4044 *
4045 * This routine is similar to sfmmu_vatopfn() but will only look for
4046 * a kernel vaddr in the hash structure for the specified rehash value.
4047 * It's just an optimization for the case when pagesize for a given
4048 * va range is already known (e.g. large page heap) and we don't want
4049 * to start the search with rehash value 1 as sfmmu_vatopfn() does.
4050 *
4051 * Returns valid pfn or PFN_INVALID if
4052 * tte for specified rehash # is not found, invalid or suspended.
4053 */
4054 ENTRY_NP(sfmmu_kvaszc2pfn)
4055 /*
4056 * disable interrupts
4057 */
4058 rdpr %pstate, %o3
4059 #ifdef DEBUG
4060 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
4061 #endif
4062 /*
4063 * disable interrupts to protect the TSBMISS area
4064 */
4065 andn %o3, PSTATE_IE, %o5
4066 wrpr %o5, 0, %pstate
4067
4068 CPU_TSBMISS_AREA(%g1, %o5)
4069 ldn [%g1 + TSBMISS_KHATID], %o4
4070 sll %o1, 1, %g6
4071 add %g6, %o1, %g6
4072 add %g6, MMU_PAGESHIFT, %g6
4073 /*
4074 * %o0 = vaddr
4075 * %o1 = hashno
4076 * %o3 = old %pstate
4077 * %o4 = ksfmmup
4078 * %g1 = tsbmiss area
4079 * %g6 = hmeshift
4080 */
4081
4082 /*
4083 * The first arg to GET_TTE is actually tagaccess register
4084 * not just vaddr. Since this call is for kernel we need to clear
4085 * any lower vaddr bits that would be interpreted as ctx bits.
4086 */
4087 srlx %o0, MMU_PAGESHIFT, %o0
4088 sllx %o0, MMU_PAGESHIFT, %o0
4089 GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5,
4090 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
4091 kvaszc2pfn_nohblk)
4092
4093 kvaszc2pfn_hblk_found:
4094 /*
4095 * %g3 = tte
4096 * %o0 = vaddr
4097 */
4098 brgez,a,pn %g3, 1f /* check if tte is invalid */
4099 mov -1, %o0 /* output = -1 (PFN_INVALID) */
4100 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
4101 /*
4102 * g3 = pfn
4103 */
4104 ba,pt %xcc, 1f
4105 mov %g3, %o0
4106
4107 kvaszc2pfn_nohblk:
4108 mov -1, %o0
4109
4110 1:
4111 retl
4112 wrpr %g0, %o3, %pstate /* re-enable interrupts */
4113
4114 SET_SIZE(sfmmu_kvaszc2pfn)
4115
4116 #endif /* lint */
4117
4118
4119
4120 #if !defined(lint)
4121
4122 /*
4123 * kpm lock used between trap level tsbmiss handler and kpm C level.
4124 */
4125 #define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \
4126 mov 0xff, tmp1 ;\
4127 label1: ;\
4128 casa [kpmlckp]asi, %g0, tmp1 ;\
4129 brnz,pn tmp1, label1 ;\
4130 mov 0xff, tmp1 ;\
4131 membar #LoadLoad
4132
4133 #define KPMLOCK_EXIT(kpmlckp, asi) \
4134 membar #LoadStore|#StoreStore ;\
4135 sta %g0, [kpmlckp]asi
4136
4137 /*
4138 * Lookup a memseg for a given pfn and if found, return the physical
4139 * address of the corresponding struct memseg in mseg, otherwise
4140 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
4141 * tsbmp, %asi is assumed to be ASI_MEM.
4142 * This lookup is done by strictly traversing only the physical memseg
4143 * linkage. The more generic approach, to check the virtual linkage
4144 * before using the physical (used e.g. with hmehash buckets), cannot
4145 * be used here. Memory DR operations can run in parallel to this
4146 * lookup w/o any locks and updates of the physical and virtual linkage
4147 * cannot be done atomically wrt. to each other. Because physical
4148 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
4149 * as "physical NULL" pointer.
4150 */
4151 #define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
4152 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\
4153 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\
4154 udivx pfn, mseg, mseg ;\
4155 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\
4156 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\
4157 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\
4158 add tmp1, mseg, tmp1 ;\
4159 ldxa [tmp1]%asi, mseg ;\
4160 cmp mseg, MSEG_NULLPTR_PA ;\
4161 be,pn %xcc, label/**/1 /* if not found */ ;\
4162 nop ;\
4163 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\
4164 cmp pfn, tmp1 /* pfn - pages_base */ ;\
4165 blu,pn %xcc, label/**/1 ;\
4166 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\
4167 cmp pfn, tmp2 /* pfn - pages_end */ ;\
4168 bgeu,pn %xcc, label/**/1 ;\
4169 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\
4170 mulx tmp1, PAGE_SIZE, tmp1 ;\
4171 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\
4172 add tmp2, tmp1, tmp1 /* pp */ ;\
4173 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\
4174 cmp tmp2, pfn ;\
4175 be,pt %xcc, label/**/_ok /* found */ ;\
4176 label/**/1: ;\
4177 /* brute force lookup */ ;\
4178 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\
4179 ldx [tmp3 + %lo(memsegspa)], mseg ;\
4180 label/**/2: ;\
4181 cmp mseg, MSEG_NULLPTR_PA ;\
4182 be,pn %xcc, label/**/_ok /* if not found */ ;\
4183 nop ;\
4184 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\
4185 cmp pfn, tmp1 /* pfn - pages_base */ ;\
4186 blu,a,pt %xcc, label/**/2 ;\
4187 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\
4188 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\
4189 cmp pfn, tmp2 /* pfn - pages_end */ ;\
4190 bgeu,a,pt %xcc, label/**/2 ;\
4191 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\
4192 label/**/_ok:
4193
4194 /*
4195 * kpm tsb miss handler large pages
4196 * g1 = 8K kpm TSB entry pointer
4197 * g2 = tag access register
4198 * g3 = 4M kpm TSB entry pointer
4199 */
4200 ALTENTRY(sfmmu_kpm_dtsb_miss)
4201 TT_TRACE(trace_tsbmiss)
4202
4203 CPU_INDEX(%g7, %g6)
4204 sethi %hi(kpmtsbm_area), %g6
4205 sllx %g7, KPMTSBM_SHIFT, %g7
4206 or %g6, %lo(kpmtsbm_area), %g6
4207 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */
4208
4209 /* check enable flag */
4210 ldub [%g6 + KPMTSBM_FLAGS], %g4
4211 and %g4, KPMTSBM_ENABLE_FLAG, %g5
4212 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */
4213 nop
4214
4215 /* VA range check */
4216 ldx [%g6 + KPMTSBM_VBASE], %g7
4217 cmp %g2, %g7
4218 blu,pn %xcc, sfmmu_tsb_miss
4219 ldx [%g6 + KPMTSBM_VEND], %g5
4220 cmp %g2, %g5
4221 bgeu,pn %xcc, sfmmu_tsb_miss
4222 stx %g3, [%g6 + KPMTSBM_TSBPTR]
4223
4224 /*
4225 * check TL tsbmiss handling flag
4226 * bump tsbmiss counter
4227 */
4228 lduw [%g6 + KPMTSBM_TSBMISS], %g5
4229 #ifdef DEBUG
4230 and %g4, KPMTSBM_TLTSBM_FLAG, %g3
4231 inc %g5
4232 brz,pn %g3, sfmmu_kpm_exception
4233 st %g5, [%g6 + KPMTSBM_TSBMISS]
4234 #else
4235 inc %g5
4236 st %g5, [%g6 + KPMTSBM_TSBMISS]
4237 #endif
4238 /*
4239 * At this point:
4240 * g1 = 8K kpm TSB pointer (not used)
4241 * g2 = tag access register
4242 * g3 = clobbered
4243 * g6 = per-CPU kpm tsbmiss area
4244 * g7 = kpm_vbase
4245 */
4246
4247 /* vaddr2pfn */
4248 ldub [%g6 + KPMTSBM_SZSHIFT], %g3
4249 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */
4250 srax %g4, %g3, %g2 /* which alias range (r) */
4251 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */
4252 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */
4253
4254 /*
4255 * Setup %asi
4256 * mseg_pa = page_numtomemseg_nolock(pfn)
4257 * if (mseg_pa == NULL) sfmmu_kpm_exception
4258 * g2=pfn
4259 */
4260 mov ASI_MEM, %asi
4261 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4262 cmp %g3, MSEG_NULLPTR_PA
4263 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */
4264 nop
4265
4266 /*
4267 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4268 * g2=pfn g3=mseg_pa
4269 */
4270 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5
4271 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4272 srlx %g2, %g5, %g4
4273 sllx %g4, %g5, %g4
4274 sub %g4, %g7, %g4
4275 srlx %g4, %g5, %g4
4276
4277 /*
4278 * Validate inx value
4279 * g2=pfn g3=mseg_pa g4=inx
4280 */
4281 #ifdef DEBUG
4282 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4283 cmp %g4, %g5 /* inx - nkpmpgs */
4284 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */
4285 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4286 #else
4287 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4288 #endif
4289 /*
4290 * kp = &mseg_pa->kpm_pages[inx]
4291 */
4292 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */
4293 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4294 add %g5, %g4, %g5 /* kp */
4295
4296 /*
4297 * KPMP_HASH(kp)
4298 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4299 */
4300 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */
4301 sub %g7, 1, %g7 /* mask */
4302 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */
4303 add %g5, %g1, %g5 /* y = ksp + x */
4304 and %g5, %g7, %g5 /* hashinx = y & mask */
4305
4306 /*
4307 * Calculate physical kpm_page pointer
4308 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4309 */
4310 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4311 add %g1, %g4, %g1 /* kp_pa */
4312
4313 /*
4314 * Calculate physical hash lock address
4315 * g1=kp_refcntc_pa g2=pfn g5=hashinx
4316 */
4317 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4318 sllx %g5, KPMHLK_SHIFT, %g5
4319 add %g4, %g5, %g3
4320 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */
4321
4322 /*
4323 * Assemble tte
4324 * g1=kp_pa g2=pfn g3=hlck_pa
4325 */
4326 #ifdef sun4v
4327 sethi %hi(TTE_VALID_INT), %g5 /* upper part */
4328 sllx %g5, 32, %g5
4329 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4330 or %g4, TTE4M, %g4
4331 or %g5, %g4, %g5
4332 #else
4333 sethi %hi(TTE_VALID_INT), %g4
4334 mov TTE4M, %g5
4335 sllx %g5, TTE_SZ_SHFT_INT, %g5
4336 or %g5, %g4, %g5 /* upper part */
4337 sllx %g5, 32, %g5
4338 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4339 or %g5, %g4, %g5
4340 #endif
4341 sllx %g2, MMU_PAGESHIFT, %g4
4342 or %g5, %g4, %g5 /* tte */
4343 ldx [%g6 + KPMTSBM_TSBPTR], %g4
4344 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */
4345
4346 /*
4347 * tsb dropin
4348 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4349 */
4350
4351 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4352 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4353
4354 /* use C-handler if there's no go for dropin */
4355 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4356 cmp %g7, -1
4357 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */
4358 nop
4359
4360 #ifdef DEBUG
4361 /* double check refcnt */
4362 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7
4363 brz,pn %g7, 5f /* let C-handler deal with this */
4364 nop
4365 #endif
4366
4367 #ifndef sun4v
4368 ldub [%g6 + KPMTSBM_FLAGS], %g7
4369 mov ASI_N, %g1
4370 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0
4371 movnz %icc, ASI_MEM, %g1
4372 mov %g1, %asi
4373 #endif
4374
4375 /*
4376 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4377 * If we fail to lock the TSB entry then just load the tte into the
4378 * TLB.
4379 */
4380 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1)
4381
4382 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4383 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4384 locked_tsb_l1:
4385 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4386
4387 /* KPMLOCK_EXIT(kpmlckp, asi) */
4388 KPMLOCK_EXIT(%g3, ASI_MEM)
4389
4390 /*
4391 * If trapstat is running, we need to shift the %tpc and %tnpc to
4392 * point to trapstat's TSB miss return code (note that trapstat
4393 * itself will patch the correct offset to add).
4394 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4395 */
4396 rdpr %tl, %g7
4397 cmp %g7, 1
4398 ble %icc, 0f
4399 sethi %hi(KERNELBASE), %g6
4400 rdpr %tpc, %g7
4401 or %g6, %lo(KERNELBASE), %g6
4402 cmp %g7, %g6
4403 bgeu %xcc, 0f
4404 ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4405 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */
4406 wrpr %g7, %tpc
4407 add %g7, 4, %g7
4408 wrpr %g7, %tnpc
4409 0:
4410 retry
4411 5:
4412 /* g3=hlck_pa */
4413 KPMLOCK_EXIT(%g3, ASI_MEM)
4414 ba,pt %icc, sfmmu_kpm_exception
4415 nop
4416 SET_SIZE(sfmmu_kpm_dtsb_miss)
4417
4418 /*
4419 * kpm tsbmiss handler for smallpages
4420 * g1 = 8K kpm TSB pointer
4421 * g2 = tag access register
4422 * g3 = 4M kpm TSB pointer
4423 */
4424 ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4425 TT_TRACE(trace_tsbmiss)
4426 CPU_INDEX(%g7, %g6)
4427 sethi %hi(kpmtsbm_area), %g6
4428 sllx %g7, KPMTSBM_SHIFT, %g7
4429 or %g6, %lo(kpmtsbm_area), %g6
4430 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */
4431
4432 /* check enable flag */
4433 ldub [%g6 + KPMTSBM_FLAGS], %g4
4434 and %g4, KPMTSBM_ENABLE_FLAG, %g5
4435 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */
4436 nop
4437
4438 /*
4439 * VA range check
4440 * On fail: goto sfmmu_tsb_miss
4441 */
4442 ldx [%g6 + KPMTSBM_VBASE], %g7
4443 cmp %g2, %g7
4444 blu,pn %xcc, sfmmu_tsb_miss
4445 ldx [%g6 + KPMTSBM_VEND], %g5
4446 cmp %g2, %g5
4447 bgeu,pn %xcc, sfmmu_tsb_miss
4448 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */
4449
4450 /*
4451 * check TL tsbmiss handling flag
4452 * bump tsbmiss counter
4453 */
4454 lduw [%g6 + KPMTSBM_TSBMISS], %g5
4455 #ifdef DEBUG
4456 and %g4, KPMTSBM_TLTSBM_FLAG, %g1
4457 inc %g5
4458 brz,pn %g1, sfmmu_kpm_exception
4459 st %g5, [%g6 + KPMTSBM_TSBMISS]
4460 #else
4461 inc %g5
4462 st %g5, [%g6 + KPMTSBM_TSBMISS]
4463 #endif
4464 /*
4465 * At this point:
4466 * g1 = clobbered
4467 * g2 = tag access register
4468 * g3 = 4M kpm TSB pointer (not used)
4469 * g6 = per-CPU kpm tsbmiss area
4470 * g7 = kpm_vbase
4471 */
4472
4473 /*
4474 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr)
4475 * which is defined in mach_kpm.h. Any changes in that macro
4476 * should also be ported back to this assembly code.
4477 */
4478 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */
4479 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */
4480 srax %g4, %g3, %g7 /* which alias range (r) */
4481 brz,pt %g7, 2f
4482 sethi %hi(vac_colors_mask), %g5
4483 ld [%g5 + %lo(vac_colors_mask)], %g5
4484
4485 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */
4486 and %g1, %g5, %g1 /* g1 = v */
4487 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */
4488 cmp %g7, %g1 /* if (r > v) */
4489 bleu,pn %xcc, 1f
4490 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */
4491 sub %g7, %g1, %g5 /* g5 = r - v */
4492 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */
4493 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */
4494 ba 2f
4495 nop
4496 1:
4497 sllx %g7, MMU_PAGESHIFT, %g5 /* else */
4498 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */
4499
4500 /*
4501 * paddr2pfn
4502 * g1 = vcolor (not used)
4503 * g2 = tag access register
4504 * g3 = clobbered
4505 * g4 = paddr
4506 * g5 = clobbered
4507 * g6 = per-CPU kpm tsbmiss area
4508 * g7 = clobbered
4509 */
4510 2:
4511 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */
4512
4513 /*
4514 * Setup %asi
4515 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4516 * if (mseg not found) sfmmu_kpm_exception
4517 * g2=pfn g6=per-CPU kpm tsbmiss area
4518 * g4 g5 g7 for scratch use.
4519 */
4520 mov ASI_MEM, %asi
4521 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4522 cmp %g3, MSEG_NULLPTR_PA
4523 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */
4524 nop
4525
4526 /*
4527 * inx = pfn - mseg_pa->kpm_pbase
4528 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area
4529 */
4530 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4531 sub %g2, %g7, %g4
4532
4533 #ifdef DEBUG
4534 /*
4535 * Validate inx value
4536 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area
4537 */
4538 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4539 cmp %g4, %g5 /* inx - nkpmpgs */
4540 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */
4541 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4542 #else
4543 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4544 #endif
4545 /* ksp = &mseg_pa->kpm_spages[inx] */
4546 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4547 add %g5, %g4, %g5 /* ksp */
4548
4549 /*
4550 * KPMP_SHASH(kp)
4551 * g2=pfn g3=mseg_pa g4=inx g5=ksp
4552 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz
4553 */
4554 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */
4555 sub %g7, 1, %g7 /* mask */
4556 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */
4557 add %g5, %g1, %g5 /* y = ksp + x */
4558 and %g5, %g7, %g5 /* hashinx = y & mask */
4559
4560 /*
4561 * Calculate physical kpm_spage pointer
4562 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4563 * g6=per-CPU kpm tsbmiss area
4564 */
4565 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4566 add %g1, %g4, %g1 /* ksp_pa */
4567
4568 /*
4569 * Calculate physical hash lock address.
4570 * Note: Changes in kpm_shlk_t must be reflected here.
4571 * g1=ksp_pa g2=pfn g5=hashinx
4572 * g6=per-CPU kpm tsbmiss area
4573 */
4574 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4575 sllx %g5, KPMSHLK_SHIFT, %g5
4576 add %g4, %g5, %g3 /* hlck_pa */
4577
4578 /*
4579 * Assemble non-cacheable tte initially
4580 * g1=ksp_pa g2=pfn g3=hlck_pa
4581 * g6=per-CPU kpm tsbmiss area
4582 */
4583 sethi %hi(TTE_VALID_INT), %g5 /* upper part */
4584 sllx %g5, 32, %g5
4585 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4586 or %g5, %g4, %g5
4587 sllx %g2, MMU_PAGESHIFT, %g4
4588 or %g5, %g4, %g5 /* tte */
4589 ldx [%g6 + KPMTSBM_TSBPTR], %g4
4590 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */
4591
4592 /*
4593 * tsb dropin
4594 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable)
4595 * g6=per-CPU kpm tsbmiss area g7=scratch register
4596 */
4597
4598 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4599 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4600
4601 /* use C-handler if there's no go for dropin */
4602 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
4603 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */
4604 bz,pt %icc, 5f /* no go */
4605 nop
4606 and %g7, KPM_MAPPED_MASK, %g7 /* go */
4607 cmp %g7, KPM_MAPPEDS /* cacheable ? */
4608 be,a,pn %xcc, 3f
4609 or %g5, TTE_CV_INT, %g5 /* cacheable */
4610 3:
4611 #ifndef sun4v
4612 ldub [%g6 + KPMTSBM_FLAGS], %g7
4613 mov ASI_N, %g1
4614 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0
4615 movnz %icc, ASI_MEM, %g1
4616 mov %g1, %asi
4617 #endif
4618
4619 /*
4620 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4621 * If we fail to lock the TSB entry then just load the tte into the
4622 * TLB.
4623 */
4624 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2)
4625
4626 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4627 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4628 locked_tsb_l2:
4629 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4630
4631 /* KPMLOCK_EXIT(kpmlckp, asi) */
4632 KPMLOCK_EXIT(%g3, ASI_MEM)
4633
4634 /*
4635 * If trapstat is running, we need to shift the %tpc and %tnpc to
4636 * point to trapstat's TSB miss return code (note that trapstat
4637 * itself will patch the correct offset to add).
4638 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4639 */
4640 rdpr %tl, %g7
4641 cmp %g7, 1
4642 ble %icc, 0f
4643 sethi %hi(KERNELBASE), %g6
4644 rdpr %tpc, %g7
4645 or %g6, %lo(KERNELBASE), %g6
4646 cmp %g7, %g6
4647 bgeu %xcc, 0f
4648 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4649 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */
4650 wrpr %g7, %tpc
4651 add %g7, 4, %g7
4652 wrpr %g7, %tnpc
4653 0:
4654 retry
4655 5:
4656 /* g3=hlck_pa */
4657 KPMLOCK_EXIT(%g3, ASI_MEM)
4658 ba,pt %icc, sfmmu_kpm_exception
4659 nop
4660 SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4661
4662 #if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4663 #error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4664 #endif
4665
4666 #endif /* lint */
4667
4668 #ifdef lint
4669 /*
4670 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
4671 * Called from C-level, sets/clears "go" indication for trap level handler.
4672 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
4673 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
4674 * Assumes khl_mutex is held when called from C-level.
4675 */
4676 /* ARGSUSED */
4677 void
4678 sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
4679 {
4680 }
4681
4682 /*
4683 * kpm_smallpages: stores val to byte at address mapped within
4684 * low level lock brackets. The old value is returned.
4685 * Called from C-level.
4686 */
4687 /* ARGSUSED */
4688 int
4689 sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val)
4690 {
4691 return (0);
4692 }
4693
4694 #else /* lint */
4695
4696 .seg ".data"
4697 sfmmu_kpm_tsbmtl_panic:
4698 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled"
4699 .byte 0
4700 sfmmu_kpm_stsbmtl_panic:
4701 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled"
4702 .byte 0
4703 .align 4
4704 .seg ".text"
4705
4706 ENTRY_NP(sfmmu_kpm_tsbmtl)
4707 rdpr %pstate, %o3
4708 /*
4709 * %o0 = &kp_refcntc
4710 * %o1 = &khl_lock
4711 * %o2 = 0/1 (off/on)
4712 * %o3 = pstate save
4713 */
4714 #ifdef DEBUG
4715 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */
4716 bnz,pt %icc, 1f /* disabled, panic */
4717 nop
4718 save %sp, -SA(MINFRAME), %sp
4719 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0
4720 call panic
4721 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4722 ret
4723 restore
4724 1:
4725 #endif /* DEBUG */
4726 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */
4727
4728 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4729 mov -1, %o5
4730 brz,a %o2, 2f
4731 mov 0, %o5
4732 2:
4733 sth %o5, [%o0]
4734 KPMLOCK_EXIT(%o1, ASI_N)
4735
4736 retl
4737 wrpr %g0, %o3, %pstate /* enable interrupts */
4738 SET_SIZE(sfmmu_kpm_tsbmtl)
4739
4740 ENTRY_NP(sfmmu_kpm_stsbmtl)
4741 rdpr %pstate, %o3
4742 /*
4743 * %o0 = &mapped
4744 * %o1 = &kshl_lock
4745 * %o2 = val
4746 * %o3 = pstate save
4747 */
4748 #ifdef DEBUG
4749 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */
4750 bnz,pt %icc, 1f /* disabled, panic */
4751 nop
4752 save %sp, -SA(MINFRAME), %sp
4753 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0
4754 call panic
4755 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4756 ret
4757 restore
4758 1:
4759 #endif /* DEBUG */
4760 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */
4761
4762 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4763 ldsb [%o0], %o5
4764 stb %o2, [%o0]
4765 KPMLOCK_EXIT(%o1, ASI_N)
4766
4767 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */
4768 retl
4769 wrpr %g0, %o3, %pstate /* enable interrupts */
4770 SET_SIZE(sfmmu_kpm_stsbmtl)
4771
4772 #endif /* lint */
4773
4774 #ifndef lint
4775 #ifdef sun4v
4776 /*
4777 * User/kernel data miss w// multiple TSBs
4778 * The first probe covers 8K, 64K, and 512K page sizes,
4779 * because 64K and 512K mappings are replicated off 8K
4780 * pointer. Second probe covers 4M page size only.
4781 *
4782 * MMU fault area contains miss address and context.
4783 */
4784 ALTENTRY(sfmmu_slow_dmmu_miss)
4785 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type
4786
4787 slow_miss_common:
4788 /*
4789 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4790 * %g3 = ctx (cannot be INVALID_CONTEXT)
4791 */
4792 brnz,pt %g3, 8f ! check for user context
4793 nop
4794
4795 /*
4796 * Kernel miss
4797 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4798 * branch to sfmmu_tsb_miss_tt to handle it.
4799 */
4800 mov %g2, %g7 ! TSB pointer macro clobbers tagacc
4801 sfmmu_dslow_patch_ktsb_base:
4802 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase
4803 sfmmu_dslow_patch_ktsb_szcode:
4804 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched)
4805
4806 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4807 ! %g1 = First TSB entry pointer, as TSB miss handler expects
4808
4809 mov %g2, %g7 ! TSB pointer macro clobbers tagacc
4810 sfmmu_dslow_patch_ktsb4m_base:
4811 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase
4812 sfmmu_dslow_patch_ktsb4m_szcode:
4813 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched)
4814
4815 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4816 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4817 ba,a,pt %xcc, sfmmu_tsb_miss_tt
4818 .empty
4819
4820 8:
4821 /*
4822 * User miss
4823 * Get first TSB pointer in %g1
4824 * Get second TSB pointer (or NULL if no second TSB) in %g3
4825 * Branch to sfmmu_tsb_miss_tt to handle it
4826 */
4827 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4828 /* %g1 = first TSB entry ptr now, %g2 preserved */
4829
4830 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */
4831 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */
4832 nop
4833
4834 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4835 /* %g3 = second TSB entry ptr now, %g2 preserved */
4836 9:
4837 ba,a,pt %xcc, sfmmu_tsb_miss_tt
4838 .empty
4839 SET_SIZE(sfmmu_slow_dmmu_miss)
4840
4841
4842 /*
4843 * User/kernel instruction miss w/ multiple TSBs
4844 * The first probe covers 8K, 64K, and 512K page sizes,
4845 * because 64K and 512K mappings are replicated off 8K
4846 * pointer. Second probe covers 4M page size only.
4847 *
4848 * MMU fault area contains miss address and context.
4849 */
4850 ALTENTRY(sfmmu_slow_immu_miss)
4851 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
4852 ba,a,pt %xcc, slow_miss_common
4853 SET_SIZE(sfmmu_slow_immu_miss)
4854
4855 #endif /* sun4v */
4856 #endif /* lint */
4857
4858 #ifndef lint
4859
4860 /*
4861 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4862 */
4863 .seg ".data"
4864 .align 64
4865 .global tsbmiss_area
4866 tsbmiss_area:
4867 .skip (TSBMISS_SIZE * NCPU)
4868
4869 .align 64
4870 .global kpmtsbm_area
4871 kpmtsbm_area:
4872 .skip (KPMTSBM_SIZE * NCPU)
4873 #endif /* lint */