1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/asm_linkage.h> 30 #include <sys/machthread.h> 31 #include <sys/privregs.h> 32 #include <sys/machasi.h> 33 #include <sys/trap.h> 34 #include <sys/mmu.h> 35 #include <sys/machparam.h> 36 #include <sys/machtrap.h> 37 #include <sys/traptrace.h> 38 39 #if !defined(lint) 40 #include "assym.h" 41 42 /* 43 * Spill fault handlers 44 * sn0 - spill normal tl 0 45 * sn1 - spill normal tl >0 46 * so0 - spill other tl 0 47 * so1 - spill other tl >0 48 */ 49 50 ENTRY_NP(fault_32bit_sn0) 51 ! 52 FAULT_WINTRACE(%g1, %g2, %g3, TT_F32_SN0) 53 ! 54 ! Spill normal tl0 fault. 55 ! This happens when a user tries to spill to an unmapped or 56 ! misaligned stack. We handle an unmapped stack by simulating 57 ! a pagefault at the trap pc and a misaligned stack by generating 58 ! a user alignment trap. 59 ! 60 ! spill the window into wbuf slot 0 61 ! (we know wbuf is empty since we came from user mode) 62 ! 63 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 64 ! sfar (g5 == T_ALIGNMENT) 65 ! 66 CPU_ADDR(%g4, %g1) 67 ldn [%g4 + CPU_MPCB], %g1 68 stn %sp, [%g1 + MPCB_SPBUF] 69 ldn [%g1 + MPCB_WBUF], %g2 70 SAVE_V8WINDOW(%g2) 71 mov 1, %g2 72 st %g2, [%g1 + MPCB_WBCNT] 73 saved 74 ! 75 ! setup user_trap args 76 ! 77 set sfmmu_tsbmiss_exception, %g1 78 mov %g6, %g2 ! arg2 = tagaccess 79 mov T_WIN_OVERFLOW, %g3 ! arg3 = traptype 80 cmp %g5, T_ALIGNMENT 81 bne %icc, 1f 82 nop 83 set trap, %g1 84 mov T_ALIGNMENT, %g3 85 1: 86 sub %g0, 1, %g4 87 ! 88 ! spill traps increment %cwp by 2, 89 ! but user_trap wants the trap %cwp 90 ! 91 rdpr %tstate, %g5 92 and %g5, TSTATE_CWP, %g5 93 ba,pt %xcc, user_trap 94 wrpr %g0, %g5, %cwp 95 SET_SIZE(fault_32bit_sn0) 96 97 ! 98 ! Spill normal tl1 fault. 99 ! This happens when sys_trap's save spills to an unmapped stack. 100 ! We handle it by spilling the window to the wbuf and trying 101 ! sys_trap again. 102 ! 103 ! spill the window into wbuf slot 0 104 ! (we know wbuf is empty since we came from user mode) 105 ! 106 ENTRY_NP(fault_32bit_sn1) 107 FAULT_WINTRACE(%g5, %g6, %g7, TT_F32_SN1) 108 CPU_PADDR(%g5, %g6) 109 mov ASI_MEM, %asi 110 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 111 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 112 stna %sp, [%g6 + MPCB_SPBUF]%asi 113 SAVE_V8WINDOW_ASI(%g5) 114 mov 1, %g5 115 sta %g5, [%g6 + MPCB_WBCNT]%asi 116 saved 117 set sys_trap, %g5 118 wrpr %g5, %tnpc 119 done 120 SET_SIZE(fault_32bit_sn1) 121 122 ENTRY_NP(fault_32bit_so0) 123 ! 124 FAULT_WINTRACE(%g5, %g6, %g1, TT_F32_SO0) 125 ! 126 ! Spill other tl0 fault. 127 ! This happens when the kernel spills a user window and that 128 ! user's stack has been unmapped. 129 ! We handle it by spilling the window into the user's wbuf. 130 ! 131 ! find lwp & increment wbcnt 132 ! 133 CPU_ADDR(%g5, %g6) 134 ldn [%g5 + CPU_MPCB], %g1 135 ld [%g1 + MPCB_WBCNT], %g2 136 add %g2, 1, %g3 137 st %g3, [%g1 + MPCB_WBCNT] 138 ! 139 ! use previous wbcnt to spill new spbuf & wbuf 140 ! 141 sll %g2, CPTRSHIFT, %g4 ! spbuf size is sizeof (caddr_t) 142 add %g1, MPCB_SPBUF, %g3 143 stn %sp, [%g3 + %g4] 144 sll %g2, RWIN32SHIFT, %g4 145 ldn [%g1 + MPCB_WBUF], %g3 146 add %g3, %g4, %g3 147 SAVE_V8WINDOW(%g3) 148 saved 149 retry 150 SET_SIZE(fault_32bit_so0) 151 152 ! 153 ! Spill other tl1 fault. 154 ! This happens when priv_trap spills a user window and that 155 ! user's stack has been unmapped. 156 ! We handle it by spilling the window to the wbuf and retrying 157 ! the save. 158 ! 159 ENTRY_NP(fault_32bit_so1) 160 FAULT_WINTRACE(%g5, %g6, %g7, TT_F32_SO1) 161 CPU_PADDR(%g5, %g6) 162 ! 163 ! find lwp & increment wbcnt 164 ! 165 mov ASI_MEM, %asi 166 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 167 lda [%g6 + MPCB_WBCNT]%asi, %g5 168 add %g5, 1, %g7 169 sta %g7, [%g6 + MPCB_WBCNT]%asi 170 ! 171 ! use previous wbcnt to spill new spbuf & wbuf 172 ! 173 sll %g5, CPTRSHIFT, %g7 ! spbuf size is sizeof (caddr_t) 174 add %g6, %g7, %g7 175 stna %sp, [%g7 + MPCB_SPBUF]%asi 176 sll %g5, RWIN32SHIFT, %g7 177 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 178 add %g5, %g7, %g7 179 SAVE_V8WINDOW_ASI(%g7) 180 saved 181 set sys_trap, %g5 182 wrpr %g5, %tnpc 183 done 184 SET_SIZE(fault_32bit_so1) 185 186 ENTRY_NP(fault_64bit_sn0) 187 ! 188 FAULT_WINTRACE(%g1, %g2, %g3, TT_F64_SN0) 189 ! 190 ! Spill normal tl0 fault. 191 ! This happens when a user tries to spill to an unmapped or 192 ! misaligned stack. We handle an unmapped stack by simulating 193 ! a pagefault at the trap pc and a misaligned stack by generating 194 ! a user alignment trap. 195 ! 196 ! spill the window into wbuf slot 0 197 ! (we know wbuf is empty since we came from user mode) 198 ! 199 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 200 ! sfar (g5 == T_ALIGNMENT) 201 ! 202 CPU_ADDR(%g4, %g1) 203 ldn [%g4 + CPU_MPCB], %g1 204 stn %sp, [%g1 + MPCB_SPBUF] 205 ldn [%g1 + MPCB_WBUF], %g2 206 SAVE_V9WINDOW(%g2) 207 mov 1, %g2 208 st %g2, [%g1 + MPCB_WBCNT] 209 saved 210 ! 211 ! setup user_trap args 212 ! 213 set sfmmu_tsbmiss_exception, %g1 214 mov %g6, %g2 ! arg2 = tagaccess 215 mov %g5, %g3 ! arg3 = traptype 216 cmp %g5, T_ALIGNMENT 217 bne %icc, 1f 218 nop 219 set trap, %g1 220 mov T_ALIGNMENT, %g3 221 1: 222 sub %g0, 1, %g4 223 ! 224 ! spill traps increment %cwp by 2, 225 ! but user_trap wants the trap %cwp 226 ! 227 rdpr %tstate, %g5 228 and %g5, TSTATE_CWP, %g5 229 ba,pt %xcc, user_trap 230 wrpr %g0, %g5, %cwp 231 SET_SIZE(fault_64bit_sn0) 232 233 ! 234 ! Spill normal tl1 fault. 235 ! This happens when sys_trap's save spills to an unmapped stack. 236 ! We handle it by spilling the window to the wbuf and trying 237 ! sys_trap again. 238 ! 239 ! spill the window into wbuf slot 0 240 ! (we know wbuf is empty since we came from user mode) 241 ! 242 ENTRY_NP(fault_64bit_sn1) 243 FAULT_WINTRACE(%g5, %g6, %g7, TT_F64_SN1) 244 CPU_PADDR(%g5, %g6) 245 mov ASI_MEM, %asi 246 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 247 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 248 stna %sp, [%g6 + MPCB_SPBUF]%asi 249 SAVE_V9WINDOW_ASI(%g5) 250 mov 1, %g5 251 sta %g5, [%g6 + MPCB_WBCNT]%asi 252 saved 253 set sys_trap, %g5 254 wrpr %g5, %tnpc 255 done 256 SET_SIZE(fault_64bit_sn1) 257 258 ! 259 ! Spill normal kernel tl1. 260 ! 261 ! spill the kernel window into kwbuf 262 ! 263 ENTRY_NP(fault_32bit_sk) 264 FAULT_WINTRACE(%g5, %g6, %g7, TT_F32_NT1) 265 CPU_PADDR(%g5, %g6) 266 set CPU_KWBUF_SP, %g6 267 add %g5, %g6, %g6 268 mov ASI_MEM, %asi 269 stna %sp, [%g6]%asi 270 set CPU_KWBUF, %g6 271 add %g5, %g6, %g6 272 SAVE_V8WINDOW_ASI(%g6) 273 mov 1, %g6 274 add %g5, CPU_MCPU, %g5 275 #ifdef DEBUG 276 lda [%g5 + MCPU_KWBUF_FULL]%asi, %g7 277 tst %g7 278 bnz,a,pn %icc, ptl1_panic 279 mov PTL1_BAD_WTRAP, %g1 280 #endif /* DEBUG */ 281 sta %g6, [%g5 + MCPU_KWBUF_FULL]%asi 282 saved 283 retry 284 SET_SIZE(fault_32bit_sk) 285 286 ! 287 ! Spill normal kernel tl1. 288 ! 289 ! spill the kernel window into kwbuf 290 ! 291 ENTRY_NP(fault_64bit_sk) 292 ! 293 FAULT_WINTRACE(%g5, %g6, %g7, TT_F64_NT1) 294 CPU_PADDR(%g5, %g6) 295 set CPU_KWBUF_SP, %g6 296 add %g5, %g6, %g6 297 mov ASI_MEM, %asi 298 stna %sp, [%g6]%asi 299 set CPU_KWBUF, %g6 300 add %g5, %g6, %g6 301 SAVE_V9WINDOW_ASI(%g6) 302 mov 1, %g6 303 add %g5, CPU_MCPU, %g5 304 #ifdef DEBUG 305 lda [%g5 + MCPU_KWBUF_FULL]%asi, %g7 306 tst %g7 307 bnz,a,pn %icc, ptl1_panic 308 mov PTL1_BAD_WTRAP, %g1 309 #endif /* DEBUG */ 310 sta %g6, [%g5 + MCPU_KWBUF_FULL]%asi 311 saved 312 retry 313 SET_SIZE(fault_64bit_sk) 314 315 ENTRY_NP(fault_64bit_so0) 316 ! 317 FAULT_WINTRACE(%g5, %g6, %g1, TT_F64_SO0) 318 ! 319 ! Spill other tl0 fault. 320 ! This happens when the kernel spills a user window and that 321 ! user's stack has been unmapped. 322 ! We handle it by spilling the window into the user's wbuf. 323 ! 324 ! find lwp & increment wbcnt 325 ! 326 CPU_ADDR(%g5, %g6) 327 ldn [%g5 + CPU_MPCB], %g1 328 ld [%g1 + MPCB_WBCNT], %g2 329 add %g2, 1, %g3 330 st %g3, [%g1 + MPCB_WBCNT] 331 ! 332 ! use previous wbcnt to spill new spbuf & wbuf 333 ! 334 sll %g2, CPTRSHIFT, %g4 ! spbuf size is sizeof (caddr_t) 335 add %g1, MPCB_SPBUF, %g3 336 stn %sp, [%g3 + %g4] 337 sll %g2, RWIN64SHIFT, %g4 338 ldn [%g1 + MPCB_WBUF], %g3 339 add %g3, %g4, %g3 340 SAVE_V9WINDOW(%g3) 341 saved 342 retry 343 SET_SIZE(fault_64bit_so0) 344 345 ! 346 ! Spill other tl1 fault. 347 ! This happens when priv_trap spills a user window and that 348 ! user's stack has been unmapped. 349 ! We handle it by spilling the window to the wbuf and retrying 350 ! the save. 351 ! 352 ENTRY_NP(fault_64bit_so1) 353 FAULT_WINTRACE(%g5, %g6, %g7, TT_F64_SO1) 354 CPU_PADDR(%g5, %g6) 355 ! 356 ! find lwp & increment wbcnt 357 ! 358 mov ASI_MEM, %asi 359 ldxa [%g5 + CPU_MPCB_PA]%asi, %g6 360 lda [%g6 + MPCB_WBCNT]%asi, %g5 361 add %g5, 1, %g7 362 sta %g7, [%g6 + MPCB_WBCNT]%asi 363 ! 364 ! use previous wbcnt to spill new spbuf & wbuf 365 ! 366 sll %g5, CPTRSHIFT, %g7 ! spbuf size is sizeof (caddr_t) 367 add %g6, %g7, %g7 368 stna %sp, [%g7 + MPCB_SPBUF]%asi 369 sll %g5, RWIN64SHIFT, %g7 370 ldxa [%g6 + MPCB_WBUF_PA]%asi, %g5 371 add %g5, %g7, %g7 372 SAVE_V9WINDOW_ASI(%g7) 373 saved 374 set sys_trap, %g5 375 wrpr %g5, %tnpc 376 done 377 SET_SIZE(fault_64bit_so1) 378 379 /* 380 * Fill fault handlers 381 * fn0 - fill normal tl 0 382 * fn1 - fill normal tl 1 383 */ 384 385 ENTRY_NP(fault_32bit_fn0) 386 ! 387 FAULT_WINTRACE(%g1, %g2, %g3, TT_F32_FN0) 388 ! 389 .fault_fn0_common: 390 ! 391 ! Fill normal tl0 fault. 392 ! This happens when a user tries to fill to an unmapped or 393 ! misaligned stack. We handle an unmapped stack by simulating 394 ! a pagefault at the trap pc and a misaligned stack by generating 395 ! a user alignment trap. 396 ! 397 ! setup user_trap args 398 ! 399 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 400 ! sfar (g5 == T_ALIGNMENT) 401 ! 402 set sfmmu_tsbmiss_exception, %g1 403 mov %g6, %g2 ! arg2 = tagaccess 404 mov T_WIN_UNDERFLOW, %g3 405 cmp %g5, T_ALIGNMENT 406 bne %icc, 1f 407 nop 408 set trap, %g1 409 mov T_ALIGNMENT, %g3 410 1: 411 sub %g0, 1, %g4 412 ! 413 ! sys_trap wants %cwp to be the same as when the trap occured, 414 ! so set it from %tstate 415 ! 416 rdpr %tstate, %g5 417 and %g5, TSTATE_CWP, %g5 418 ba,pt %xcc, user_trap 419 wrpr %g0, %g5, %cwp 420 SET_SIZE(fault_32bit_fn0) 421 422 ENTRY_NP(fault_32bit_fn1) 423 ! 424 FAULT_WINTRACE(%g1, %g2, %g3, TT_F32_FN1) 425 ! 426 wrpr %g0, 1, %gl 427 srl %sp, 0, %g7 428 ! 429 .fault_fn1_common: 430 ! 431 ! Fill normal tl1 fault. 432 ! This happens when user_rtt's restore fills from an unmapped or 433 ! misaligned stack. We handle an unmapped stack by simulating 434 ! a pagefault at user_rtt and a misaligned stack by generating 435 ! a RTT alignment trap. 436 ! 437 ! save fault addr & fix %cwp 438 ! 439 rdpr %tstate, %g1 440 and %g1, TSTATE_CWP, %g1 441 wrpr %g0, %g1, %cwp 442 ! 443 ! fake tl1 traps regs so that after pagefault runs, we 444 ! re-execute at user_rtt. 445 ! 446 wrpr %g0, 1, %tl 447 set TSTATE_KERN | TSTATE_IE, %g1 448 wrpr %g0, %g1, %tstate 449 set user_rtt, %g1 450 wrpr %g0, %g1, %tpc 451 add %g1, 4, %g1 452 wrpr %g0, %g1, %tnpc 453 ! 454 ! setup sys_trap args 455 ! 456 ! g5 = mmu trap type, g6 = tag access reg (g5 != T_ALIGNMENT) or 457 ! sfar (g5 == T_ALIGNMENT) 458 ! 459 set sfmmu_tsbmiss_exception, %g1 460 mov %g6, %g2 ! arg2 = tagaccess 461 set T_USER | T_SYS_RTT_PAGE, %g3 ! arg3 = traptype 462 cmp %g5, T_ALIGNMENT 463 bne %icc, 1f 464 nop 465 set trap, %g1 466 set T_USER | T_SYS_RTT_ALIGN, %g3 467 1: 468 sub %g0, 1, %g4 469 ! 470 ! setup to run kernel again by setting THREAD_REG, %wstate 471 ! and the mmu to their kernel values. 472 ! 473 ! sun4v cannot safely lower %gl then raise it again 474 ! so ktl0 must restore THREAD_REG 475 rdpr %wstate, %l1 476 sllx %l1, WSTATE_SHIFT, %l1 477 wrpr %l1, WSTATE_K64, %wstate 478 mov KCONTEXT, %g5 479 mov MMU_PCONTEXT, %g6 480 stxa %g5, [%g6]ASI_MMU_CTX 481 membar #Sync 482 483 ba,pt %xcc, priv_trap 484 nop 485 SET_SIZE(fault_32bit_fn1) 486 487 ENTRY_NP(fault_64bit_fn0) 488 FAULT_WINTRACE(%g1, %g2, %g3, TT_F64_FN0) 489 b .fault_fn0_common 490 nop 491 SET_SIZE(fault_64bit_fn0) 492 493 ENTRY_NP(fault_64bit_fn1) 494 FAULT_WINTRACE(%g1, %g2, %g3, TT_F64_FN1) 495 wrpr %g0, 1, %gl 496 b .fault_fn1_common 497 nop 498 SET_SIZE(fault_64bit_fn1) 499 500 ENTRY_NP(fault_rtt_fn1) 501 FAULT_WINTRACE(%g1, %g2, %g3, TT_RTT_FN1) 502 wrpr %g0, 1, %gl 503 b .fault_fn1_common 504 nop 505 SET_SIZE(fault_rtt_fn1) 506 507 /* 508 * Kernel fault handlers 509 */ 510 ENTRY_NP(fault_32bit_not) 511 ENTRY_NP(fault_64bit_not) 512 ba,pt %xcc, ptl1_panic 513 mov PTL1_BAD_WTRAP, %g1 514 SET_SIZE(fault_32bit_not) 515 SET_SIZE(fault_64bit_not) 516 #endif /* !lint */