1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/asm_linkage.h> 28 #include <sys/machthread.h> 29 #include <sys/privregs.h> 30 #include <sys/machasi.h> 31 #include <sys/trap.h> 32 #include <sys/mmu.h> 33 #include <sys/machparam.h> 34 #include <sys/machtrap.h> 35 #include <sys/traptrace.h> 36 37 #include "assym.h" 38 39 /* 40 * Spill fault handlers 41 * sn0 - spill normal tl 0 42 * sn1 - spill normal tl >0 43 * so0 - spill other tl 0 44 * so1 - spill other tl >0 45 */ 46 47 ENTRY_NP(fault_32bit_sn0) 48 ! 49 FAULT_WINTRACE(%g1, %g2, %g3, TT_F32_SN0) 50 ! 51 ! Spill normal tl0 fault. 52 ! This happens when a user tries to spill to an unmapped or 53 ! misaligned stack. We handle an unmapped stack by simulating 54 ! a pagefault at the trap pc and a misaligned stack by generating 55 ! a user alignment trap. 56 ! 57 ! spill the window into wbuf slot 0 58 ! (we know wbuf is empty since we came from user mode) 59 ! 60 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 61 ! sfar (g5 == T_ALIGNMENT) 62 ! 63 CPU_ADDR(%g4, %g1) 64 ldn [%g4 + CPU_MPCB], %g1 65 stn %sp, [%g1 + MPCB_SPBUF] 66 ldn [%g1 + MPCB_WBUF], %g2 67 SAVE_V8WINDOW(%g2) 68 mov 1, %g2 69 st %g2, [%g1 + MPCB_WBCNT] 70 saved 71 ! 72 ! setup user_trap args 73 ! 74 set sfmmu_tsbmiss_exception, %g1 75 mov %g6, %g2 ! arg2 = tagaccess 76 mov T_WIN_OVERFLOW, %g3 ! arg3 = traptype 77 cmp %g5, T_ALIGNMENT 78 bne %icc, 1f 79 nop 80 set trap, %g1 81 mov T_ALIGNMENT, %g3 82 1: 83 sub %g0, 1, %g4 84 ! 85 ! spill traps increment %cwp by 2, 86 ! but user_trap wants the trap %cwp 87 ! 88 rdpr %tstate, %g5 89 and %g5, TSTATE_CWP, %g5 90 ba,pt %xcc, user_trap 91 wrpr %g0, %g5, %cwp 92 SET_SIZE(fault_32bit_sn0) 93 94 ! 95 ! Spill normal tl1 fault. 96 ! This happens when sys_trap's save spills to an unmapped stack. 97 ! We handle it by spilling the window to the wbuf and trying 98 ! sys_trap again. 99 ! 100 ! spill the window into wbuf slot 0 101 ! (we know wbuf is empty since we came from user mode) 102 ! 103 ENTRY_NP(fault_32bit_sn1) 104 FAULT_WINTRACE(%g5, %g6, %g7, TT_F32_SN1) 105 CPU_PADDR(%g5, %g6) 106 mov ASI_MEM, %asi 107 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 108 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 109 stna %sp, [%g6 + MPCB_SPBUF]%asi 110 SAVE_V8WINDOW_ASI(%g5) 111 mov 1, %g5 112 sta %g5, [%g6 + MPCB_WBCNT]%asi 113 saved 114 set sys_trap, %g5 115 wrpr %g5, %tnpc 116 done 117 SET_SIZE(fault_32bit_sn1) 118 119 ENTRY_NP(fault_32bit_so0) 120 ! 121 FAULT_WINTRACE(%g5, %g6, %g1, TT_F32_SO0) 122 ! 123 ! Spill other tl0 fault. 124 ! This happens when the kernel spills a user window and that 125 ! user's stack has been unmapped. 126 ! We handle it by spilling the window into the user's wbuf. 127 ! 128 ! find lwp & increment wbcnt 129 ! 130 CPU_ADDR(%g5, %g6) 131 ldn [%g5 + CPU_MPCB], %g1 132 ld [%g1 + MPCB_WBCNT], %g2 133 add %g2, 1, %g3 134 st %g3, [%g1 + MPCB_WBCNT] 135 ! 136 ! use previous wbcnt to spill new spbuf & wbuf 137 ! 138 sll %g2, CPTRSHIFT, %g4 ! spbuf size is sizeof (caddr_t) 139 add %g1, MPCB_SPBUF, %g3 140 stn %sp, [%g3 + %g4] 141 sll %g2, RWIN32SHIFT, %g4 142 ldn [%g1 + MPCB_WBUF], %g3 143 add %g3, %g4, %g3 144 SAVE_V8WINDOW(%g3) 145 saved 146 retry 147 SET_SIZE(fault_32bit_so0) 148 149 ! 150 ! Spill other tl1 fault. 151 ! This happens when priv_trap spills a user window and that 152 ! user's stack has been unmapped. 153 ! We handle it by spilling the window to the wbuf and retrying 154 ! the save. 155 ! 156 ENTRY_NP(fault_32bit_so1) 157 FAULT_WINTRACE(%g5, %g6, %g7, TT_F32_SO1) 158 CPU_PADDR(%g5, %g6) 159 ! 160 ! find lwp & increment wbcnt 161 ! 162 mov ASI_MEM, %asi 163 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 164 lda [%g6 + MPCB_WBCNT]%asi, %g5 165 add %g5, 1, %g7 166 sta %g7, [%g6 + MPCB_WBCNT]%asi 167 ! 168 ! use previous wbcnt to spill new spbuf & wbuf 169 ! 170 sll %g5, CPTRSHIFT, %g7 ! spbuf size is sizeof (caddr_t) 171 add %g6, %g7, %g7 172 stna %sp, [%g7 + MPCB_SPBUF]%asi 173 sll %g5, RWIN32SHIFT, %g7 174 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 175 add %g5, %g7, %g7 176 SAVE_V8WINDOW_ASI(%g7) 177 saved 178 set sys_trap, %g5 179 wrpr %g5, %tnpc 180 done 181 SET_SIZE(fault_32bit_so1) 182 183 ENTRY_NP(fault_64bit_sn0) 184 ! 185 FAULT_WINTRACE(%g1, %g2, %g3, TT_F64_SN0) 186 ! 187 ! Spill normal tl0 fault. 188 ! This happens when a user tries to spill to an unmapped or 189 ! misaligned stack. We handle an unmapped stack by simulating 190 ! a pagefault at the trap pc and a misaligned stack by generating 191 ! a user alignment trap. 192 ! 193 ! spill the window into wbuf slot 0 194 ! (we know wbuf is empty since we came from user mode) 195 ! 196 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 197 ! sfar (g5 == T_ALIGNMENT) 198 ! 199 CPU_ADDR(%g4, %g1) 200 ldn [%g4 + CPU_MPCB], %g1 201 stn %sp, [%g1 + MPCB_SPBUF] 202 ldn [%g1 + MPCB_WBUF], %g2 203 SAVE_V9WINDOW(%g2) 204 mov 1, %g2 205 st %g2, [%g1 + MPCB_WBCNT] 206 saved 207 ! 208 ! setup user_trap args 209 ! 210 set sfmmu_tsbmiss_exception, %g1 211 mov %g6, %g2 ! arg2 = tagaccess 212 mov %g5, %g3 ! arg3 = traptype 213 cmp %g5, T_ALIGNMENT 214 bne %icc, 1f 215 nop 216 set trap, %g1 217 mov T_ALIGNMENT, %g3 218 1: 219 sub %g0, 1, %g4 220 ! 221 ! spill traps increment %cwp by 2, 222 ! but user_trap wants the trap %cwp 223 ! 224 rdpr %tstate, %g5 225 and %g5, TSTATE_CWP, %g5 226 ba,pt %xcc, user_trap 227 wrpr %g0, %g5, %cwp 228 SET_SIZE(fault_64bit_sn0) 229 230 ! 231 ! Spill normal tl1 fault. 232 ! This happens when sys_trap's save spills to an unmapped stack. 233 ! We handle it by spilling the window to the wbuf and trying 234 ! sys_trap again. 235 ! 236 ! spill the window into wbuf slot 0 237 ! (we know wbuf is empty since we came from user mode) 238 ! 239 ENTRY_NP(fault_64bit_sn1) 240 FAULT_WINTRACE(%g5, %g6, %g7, TT_F64_SN1) 241 CPU_PADDR(%g5, %g6) 242 mov ASI_MEM, %asi 243 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 244 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 245 stna %sp, [%g6 + MPCB_SPBUF]%asi 246 SAVE_V9WINDOW_ASI(%g5) 247 mov 1, %g5 248 sta %g5, [%g6 + MPCB_WBCNT]%asi 249 saved 250 set sys_trap, %g5 251 wrpr %g5, %tnpc 252 done 253 SET_SIZE(fault_64bit_sn1) 254 255 ! 256 ! Spill normal kernel tl1. 257 ! 258 ! spill the kernel window into kwbuf 259 ! 260 ENTRY_NP(fault_32bit_sk) 261 FAULT_WINTRACE(%g5, %g6, %g7, TT_F32_NT1) 262 CPU_PADDR(%g5, %g6) 263 set CPU_KWBUF_SP, %g6 264 add %g5, %g6, %g6 265 mov ASI_MEM, %asi 266 stna %sp, [%g6]%asi 267 set CPU_KWBUF, %g6 268 add %g5, %g6, %g6 269 SAVE_V8WINDOW_ASI(%g6) 270 mov 1, %g6 271 add %g5, CPU_MCPU, %g5 272 #ifdef DEBUG 273 lda [%g5 + MCPU_KWBUF_FULL]%asi, %g7 274 tst %g7 275 bnz,a,pn %icc, ptl1_panic 276 mov PTL1_BAD_WTRAP, %g1 277 #endif /* DEBUG */ 278 sta %g6, [%g5 + MCPU_KWBUF_FULL]%asi 279 saved 280 retry 281 SET_SIZE(fault_32bit_sk) 282 283 ! 284 ! Spill normal kernel tl1. 285 ! 286 ! spill the kernel window into kwbuf 287 ! 288 ENTRY_NP(fault_64bit_sk) 289 ! 290 FAULT_WINTRACE(%g5, %g6, %g7, TT_F64_NT1) 291 CPU_PADDR(%g5, %g6) 292 set CPU_KWBUF_SP, %g6 293 add %g5, %g6, %g6 294 mov ASI_MEM, %asi 295 stna %sp, [%g6]%asi 296 set CPU_KWBUF, %g6 297 add %g5, %g6, %g6 298 SAVE_V9WINDOW_ASI(%g6) 299 mov 1, %g6 300 add %g5, CPU_MCPU, %g5 301 #ifdef DEBUG 302 lda [%g5 + MCPU_KWBUF_FULL]%asi, %g7 303 tst %g7 304 bnz,a,pn %icc, ptl1_panic 305 mov PTL1_BAD_WTRAP, %g1 306 #endif /* DEBUG */ 307 sta %g6, [%g5 + MCPU_KWBUF_FULL]%asi 308 saved 309 retry 310 SET_SIZE(fault_64bit_sk) 311 312 ENTRY_NP(fault_64bit_so0) 313 ! 314 FAULT_WINTRACE(%g5, %g6, %g1, TT_F64_SO0) 315 ! 316 ! Spill other tl0 fault. 317 ! This happens when the kernel spills a user window and that 318 ! user's stack has been unmapped. 319 ! We handle it by spilling the window into the user's wbuf. 320 ! 321 ! find lwp & increment wbcnt 322 ! 323 CPU_ADDR(%g5, %g6) 324 ldn [%g5 + CPU_MPCB], %g1 325 ld [%g1 + MPCB_WBCNT], %g2 326 add %g2, 1, %g3 327 st %g3, [%g1 + MPCB_WBCNT] 328 ! 329 ! use previous wbcnt to spill new spbuf & wbuf 330 ! 331 sll %g2, CPTRSHIFT, %g4 ! spbuf size is sizeof (caddr_t) 332 add %g1, MPCB_SPBUF, %g3 333 stn %sp, [%g3 + %g4] 334 sll %g2, RWIN64SHIFT, %g4 335 ldn [%g1 + MPCB_WBUF], %g3 336 add %g3, %g4, %g3 337 SAVE_V9WINDOW(%g3) 338 saved 339 retry 340 SET_SIZE(fault_64bit_so0) 341 342 ! 343 ! Spill other tl1 fault. 344 ! This happens when priv_trap spills a user window and that 345 ! user's stack has been unmapped. 346 ! We handle it by spilling the window to the wbuf and retrying 347 ! the save. 348 ! 349 ENTRY_NP(fault_64bit_so1) 350 FAULT_WINTRACE(%g5, %g6, %g7, TT_F64_SO1) 351 CPU_PADDR(%g5, %g6) 352 ! 353 ! find lwp & increment wbcnt 354 ! 355 mov ASI_MEM, %asi 356 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 357 lda [%g6 + MPCB_WBCNT]%asi, %g5 358 add %g5, 1, %g7 359 sta %g7, [%g6 + MPCB_WBCNT]%asi 360 ! 361 ! use previous wbcnt to spill new spbuf & wbuf 362 ! 363 sll %g5, CPTRSHIFT, %g7 ! spbuf size is sizeof (caddr_t) 364 add %g6, %g7, %g7 365 stna %sp, [%g7 + MPCB_SPBUF]%asi 366 sll %g5, RWIN64SHIFT, %g7 367 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 368 add %g5, %g7, %g7 369 SAVE_V9WINDOW_ASI(%g7) 370 saved 371 set sys_trap, %g5 372 wrpr %g5, %tnpc 373 done 374 SET_SIZE(fault_64bit_so1) 375 376 /* 377 * Fill fault handlers 378 * fn0 - fill normal tl 0 379 * fn1 - fill normal tl 1 380 */ 381 382 ENTRY_NP(fault_32bit_fn0) 383 ! 384 FAULT_WINTRACE(%g1, %g2, %g3, TT_F32_FN0) 385 ! 386 .fault_fn0_common: 387 ! 388 ! Fill normal tl0 fault. 389 ! This happens when a user tries to fill to an unmapped or 390 ! misaligned stack. We handle an unmapped stack by simulating 391 ! a pagefault at the trap pc and a misaligned stack by generating 392 ! a user alignment trap. 393 ! 394 ! setup user_trap args 395 ! 396 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 397 ! sfar (g5 == T_ALIGNMENT) 398 ! 399 set sfmmu_tsbmiss_exception, %g1 400 mov %g6, %g2 ! arg2 = tagaccess 401 mov T_WIN_UNDERFLOW, %g3 402 cmp %g5, T_ALIGNMENT 403 bne %icc, 1f 404 nop 405 set trap, %g1 406 mov T_ALIGNMENT, %g3 407 1: 408 sub %g0, 1, %g4 409 ! 410 ! sys_trap wants %cwp to be the same as when the trap occured, 411 ! so set it from %tstate 412 ! 413 rdpr %tstate, %g5 414 and %g5, TSTATE_CWP, %g5 415 ba,pt %xcc, user_trap 416 wrpr %g0, %g5, %cwp 417 SET_SIZE(fault_32bit_fn0) 418 419 ENTRY_NP(fault_32bit_fn1) 420 ! 421 FAULT_WINTRACE(%g1, %g2, %g3, TT_F32_FN1) 422 ! 423 wrpr %g0, 1, %gl 424 srl %sp, 0, %g7 425 ! 426 .fault_fn1_common: 427 ! 428 ! Fill normal tl1 fault. 429 ! This happens when user_rtt's restore fills from an unmapped or 430 ! misaligned stack. We handle an unmapped stack by simulating 431 ! a pagefault at user_rtt and a misaligned stack by generating 432 ! a RTT alignment trap. 433 ! 434 ! save fault addr & fix %cwp 435 ! 436 rdpr %tstate, %g1 437 and %g1, TSTATE_CWP, %g1 438 wrpr %g0, %g1, %cwp 439 ! 440 ! fake tl1 traps regs so that after pagefault runs, we 441 ! re-execute at user_rtt. 442 ! 443 wrpr %g0, 1, %tl 444 set TSTATE_KERN | TSTATE_IE, %g1 445 wrpr %g0, %g1, %tstate 446 set user_rtt, %g1 447 wrpr %g0, %g1, %tpc 448 add %g1, 4, %g1 449 wrpr %g0, %g1, %tnpc 450 ! 451 ! setup sys_trap args 452 ! 453 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 454 ! sfar (g5 == T_ALIGNMENT) 455 ! 456 set sfmmu_tsbmiss_exception, %g1 457 mov %g6, %g2 ! arg2 = tagaccess 458 set T_USER | T_SYS_RTT_PAGE, %g3 ! arg3 = traptype 459 cmp %g5, T_ALIGNMENT 460 bne %icc, 1f 461 nop 462 set trap, %g1 463 set T_USER | T_SYS_RTT_ALIGN, %g3 464 1: 465 sub %g0, 1, %g4 466 ! 467 ! setup to run kernel again by setting THREAD_REG, %wstate 468 ! and the mmu to their kernel values. 469 ! 470 ! sun4v cannot safely lower %gl then raise it again 471 ! so ktl0 must restore THREAD_REG 472 rdpr %wstate, %l1 473 sllx %l1, WSTATE_SHIFT, %l1 474 wrpr %l1, WSTATE_K64, %wstate 475 mov KCONTEXT, %g5 476 mov MMU_PCONTEXT, %g6 477 stxa %g5, [%g6]ASI_MMU_CTX 478 membar #Sync 479 480 ba,pt %xcc, priv_trap 481 nop 482 SET_SIZE(fault_32bit_fn1) 483 484 ENTRY_NP(fault_64bit_fn0) 485 FAULT_WINTRACE(%g1, %g2, %g3, TT_F64_FN0) 486 b .fault_fn0_common 487 nop 488 SET_SIZE(fault_64bit_fn0) 489 490 ENTRY_NP(fault_64bit_fn1) 491 FAULT_WINTRACE(%g1, %g2, %g3, TT_F64_FN1) 492 wrpr %g0, 1, %gl 493 b .fault_fn1_common 494 nop 495 SET_SIZE(fault_64bit_fn1) 496 497 ENTRY_NP(fault_rtt_fn1) 498 FAULT_WINTRACE(%g1, %g2, %g3, TT_RTT_FN1) 499 wrpr %g0, 1, %gl 500 b .fault_fn1_common 501 nop 502 SET_SIZE(fault_rtt_fn1) 503 504 /* 505 * Kernel fault handlers 506 */ 507 ENTRY_NP(fault_32bit_not) 508 ENTRY_NP(fault_64bit_not) 509 ba,pt %xcc, ptl1_panic 510 mov PTL1_BAD_WTRAP, %g1 511 SET_SIZE(fault_32bit_not) 512 SET_SIZE(fault_64bit_not)