1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/param.h> 28 #include <sys/errno.h> 29 #include <sys/asm_linkage.h> 30 #include <sys/vtrace.h> 31 #include <sys/machthread.h> 32 #include <sys/clock.h> 33 #include <sys/asi.h> 34 #include <sys/fsr.h> 35 #include <sys/privregs.h> 36 37 #include "assym.h" 38 39 #define FP_USED 1 40 #define LOFAULT_SET 2 41 42 /* 43 * Error barrier: 44 * We use membar sync to establish an error barrier for 45 * deferred errors. Membar syncs are added before any update 46 * to t_lofault to ensure that deferred errors from earlier 47 * accesses will not be reported after the membar. This error 48 * isolation is important when we try to recover from async 49 * errors which tries to distinguish kernel accesses to user 50 * data. 51 */ 52 53 /* 54 * Zero a block of storage. 55 * 56 * uzero is used by the kernel to zero a block in user address space. 57 */ 58 59 ENTRY(uzero) 60 ! 61 ! Set a new lo_fault handler only if we came in with one 62 ! already specified. 63 ! 64 wr %g0, ASI_USER, %asi 65 ldn [THREAD_REG + T_LOFAULT], %o5 66 tst %o5 67 bz,pt %ncc, .do_zero 68 sethi %hi(.zeroerr), %o2 69 or %o2, %lo(.zeroerr), %o2 70 membar #Sync 71 ba,pt %ncc, .do_zero 72 stn %o2, [THREAD_REG + T_LOFAULT] 73 74 ENTRY(kzero) 75 ! 76 ! Always set a lo_fault handler 77 ! 78 wr %g0, ASI_P, %asi 79 ldn [THREAD_REG + T_LOFAULT], %o5 80 sethi %hi(.zeroerr), %o2 81 or %o5, LOFAULT_SET, %o5 82 or %o2, %lo(.zeroerr), %o2 83 membar #Sync 84 ba,pt %ncc, .do_zero 85 stn %o2, [THREAD_REG + T_LOFAULT] 86 87 /* 88 * We got here because of a fault during kzero or if 89 * uzero or bzero was called with t_lofault non-zero. 90 * Otherwise we've already run screaming from the room. 91 * Errno value is in %g1. Note that we're here iff 92 * we did set t_lofault. 93 */ 94 .zeroerr: 95 ! 96 ! Undo asi register setting. Just set it to be the 97 ! kernel default without checking. 98 ! 99 wr %g0, ASI_P, %asi 100 ! 101 ! If saved t_lofault has FP_USED set, clear the %fprs register 102 ! 103 btst FP_USED, %o5 104 bz,pt %ncc, 1f ! skip if not used 105 nop 106 membar #Sync 107 wr %g0, %g0, %fprs ! clear fprs 108 andn %o5, FP_USED, %o5 ! turn off flag bit 109 ! 110 ! We did set t_lofault. It may well have been zero coming in. 111 ! 112 1: 113 tst %o5 114 membar #Sync 115 bne,pn %ncc, 3f 116 andncc %o5, LOFAULT_SET, %o5 117 2: 118 ! 119 ! Old handler was zero. Just return the error. 120 ! 121 retl ! return 122 mov %g1, %o0 ! error code from %g1 123 3: 124 ! 125 ! We're here because %o5 was non-zero. It was non-zero 126 ! because either LOFAULT_SET was present, a previous fault 127 ! handler was present or both. In all cases we need to reset 128 ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET 129 ! before we either simply return the error or we invoke the 130 ! previously specified handler. 131 ! 132 be %ncc, 2b 133 stn %o5, [THREAD_REG + T_LOFAULT] 134 jmp %o5 ! goto real handler 135 nop 136 SET_SIZE(kzero) 137 SET_SIZE(uzero) 138 139 /* 140 * Zero a block of storage. 141 */ 142 143 ENTRY(bzero) 144 wr %g0, ASI_P, %asi 145 146 ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector 147 tst %o5 148 bz,pt %ncc, .do_zero 149 sethi %hi(.zeroerr), %o2 150 or %o2, %lo(.zeroerr), %o2 151 membar #Sync ! sync error barrier 152 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector 153 154 .do_zero: 155 cmp %o1, 15 ! check for small counts 156 blu,pn %ncc, .byteclr ! just clear bytes 157 nop 158 159 cmp %o1, 192 ! check for large counts 160 blu %ncc, .bzero_small 161 nop 162 163 sethi %hi(use_hw_bzero), %o2 164 ld [%o2 + %lo(use_hw_bzero)], %o2 165 tst %o2 166 bz %icc, .bzero_small 167 nop 168 169 rd %fprs, %o2 ! check for unused fp 170 btst FPRS_FEF, %o2 171 bnz %icc, .bzero_small 172 nop 173 174 ldn [THREAD_REG + T_LWP], %o2 175 tst %o2 176 bz,pn %ncc, .bzero_small 177 nop 178 179 ! Check for block alignment 180 btst (64-1), %o0 181 bz %icc, .bzl_block 182 nop 183 184 ! Check for double-word alignment 185 btst (8-1), %o0 186 bz %icc, .bzl_dword 187 nop 188 189 ! Check for word alignment 190 btst (4-1), %o0 191 bz %icc, .bzl_word 192 nop 193 194 ! Clear bytes until word aligned 195 .bzl_byte: 196 stba %g0, [%o0]%asi 197 add %o0, 1, %o0 198 btst (4-1), %o0 199 bnz %icc, .bzl_byte 200 sub %o1, 1, %o1 201 202 ! Check for dword-aligned 203 btst (8-1), %o0 204 bz %icc, .bzl_dword 205 nop 206 207 ! Clear words until double-word aligned 208 .bzl_word: 209 sta %g0, [%o0]%asi 210 add %o0, 4, %o0 211 btst (8-1), %o0 212 bnz %icc, .bzl_word 213 sub %o1, 4, %o1 214 215 .bzl_dword: 216 ! Clear dwords until block aligned 217 stxa %g0, [%o0]%asi 218 add %o0, 8, %o0 219 btst (64-1), %o0 220 bnz %icc, .bzl_dword 221 sub %o1, 8, %o1 222 223 .bzl_block: 224 membar #StoreStore|#StoreLoad|#LoadStore 225 wr %g0, FPRS_FEF, %fprs 226 227 ! Set the lower bit in the saved t_lofault to indicate 228 ! that we need to clear the %fprs register on the way 229 ! out 230 or %o5, FP_USED, %o5 231 232 ! Clear block 233 fzero %d0 234 fzero %d2 235 fzero %d4 236 fzero %d6 237 fzero %d8 238 fzero %d10 239 fzero %d12 240 fzero %d14 241 rd %asi, %o3 242 wr %g0, ASI_BLK_P, %asi 243 cmp %o3, ASI_P 244 bne,a %icc, 1f 245 wr %g0, ASI_BLK_AIUS, %asi 246 1: 247 mov 256, %o3 248 ba,pt %ncc, .bzl_doblock 249 nop 250 251 .bzl_blkstart: 252 ! stda %d0, [%o0+192]%asi ! in dly slot of branch that got us here 253 stda %d0, [%o0+128]%asi 254 stda %d0, [%o0+64]%asi 255 stda %d0, [%o0]%asi 256 .bzl_zinst: 257 add %o0, %o3, %o0 258 sub %o1, %o3, %o1 259 .bzl_doblock: 260 cmp %o1, 256 261 bgeu,a %ncc, .bzl_blkstart 262 stda %d0, [%o0+192]%asi 263 264 cmp %o1, 64 265 blu %ncc, .bzl_finish 266 267 andn %o1, (64-1), %o3 268 srl %o3, 4, %o2 ! using blocks, 1 instr / 16 words 269 set .bzl_zinst, %o4 270 sub %o4, %o2, %o4 271 jmp %o4 272 nop 273 274 .bzl_finish: 275 membar #StoreLoad|#StoreStore 276 wr %g0, %g0, %fprs 277 andn %o5, FP_USED, %o5 278 279 rd %asi, %o4 280 wr %g0, ASI_P, %asi 281 cmp %o4, ASI_BLK_P 282 bne,a %icc, 1f 283 wr %g0, ASI_USER, %asi 284 1: 285 286 .bzlf_dword: 287 ! double words 288 cmp %o1, 8 289 blu %ncc, .bzlf_word 290 nop 291 stxa %g0, [%o0]%asi 292 add %o0, 8, %o0 293 sub %o1, 8, %o1 294 ba,pt %ncc, .bzlf_dword 295 nop 296 297 .bzlf_word: 298 ! words 299 cmp %o1, 4 300 blu %ncc, .bzlf_byte 301 nop 302 sta %g0, [%o0]%asi 303 add %o0, 4, %o0 304 sub %o1, 4, %o1 305 ba,pt %ncc, .bzlf_word 306 nop 307 308 1: 309 add %o0, 1, %o0 ! increment address 310 .bzlf_byte: 311 subcc %o1, 1, %o1 ! decrement count 312 bgeu,a %ncc, 1b 313 stba %g0, [%o0]%asi ! zero a byte 314 315 ! 316 ! If we used the FP registers, that bit was turned 317 ! off after we were finished. We're just concerned with 318 ! whether t_lofault was set when we came in. We end up 319 ! here from either kzero() or bzero(). kzero() *always* 320 ! sets a lofault handler. It ors LOFAULT_SET into %o5 321 ! to indicate it has done this even if the value of %o5 322 ! is otherwise zero. bzero() sets a lofault handler *only* 323 ! if one was previously set. Accordingly we need to examine 324 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET 325 ! before resetting the error handler. 326 ! 327 tst %o5 328 bz,pt %ncc, 1f 329 andn %o5, LOFAULT_SET, %o5 330 membar #Sync ! sync error barrier 331 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 332 1: 333 retl 334 clr %o0 ! return (0) 335 336 .bzero_small: 337 338 ! 339 ! Check for word alignment. 340 ! 341 btst 3, %o0 342 bz .bzero_probe 343 mov 0x100, %o3 ! constant size of main loop 344 ! 345 ! 346 ! clear bytes until word aligned 347 ! 348 1: stba %g0,[%o0]%asi 349 add %o0, 1, %o0 350 btst 3, %o0 351 bnz 1b 352 sub %o1, 1, %o1 353 .bzero_probe: 354 355 ! 356 ! if needed move a word to become double-word aligned. 357 ! 358 btst 7, %o0 ! is double aligned? 359 bz %icc, .bzero_nobuf 360 nop 361 sta %g0, [%o0]%asi ! clr to double boundry 362 sub %o1, 4, %o1 363 ba,pt %ncc, .bzero_nobuf 364 add %o0, 4, %o0 365 366 !stxa %g0, [%o0+0xf8]%asi 367 .bzero_blk: 368 stxa %g0, [%o0+0xf0]%asi 369 stxa %g0, [%o0+0xe8]%asi 370 stxa %g0, [%o0+0xe0]%asi 371 stxa %g0, [%o0+0xd8]%asi 372 stxa %g0, [%o0+0xd0]%asi 373 stxa %g0, [%o0+0xc8]%asi 374 stxa %g0, [%o0+0xc0]%asi 375 stxa %g0, [%o0+0xb8]%asi 376 stxa %g0, [%o0+0xb0]%asi 377 stxa %g0, [%o0+0xa8]%asi 378 stxa %g0, [%o0+0xa0]%asi 379 stxa %g0, [%o0+0x98]%asi 380 stxa %g0, [%o0+0x90]%asi 381 stxa %g0, [%o0+0x88]%asi 382 stxa %g0, [%o0+0x80]%asi 383 stxa %g0, [%o0+0x78]%asi 384 stxa %g0, [%o0+0x70]%asi 385 stxa %g0, [%o0+0x68]%asi 386 stxa %g0, [%o0+0x60]%asi 387 stxa %g0, [%o0+0x58]%asi 388 stxa %g0, [%o0+0x50]%asi 389 stxa %g0, [%o0+0x48]%asi 390 stxa %g0, [%o0+0x40]%asi 391 stxa %g0, [%o0+0x38]%asi 392 stxa %g0, [%o0+0x30]%asi 393 stxa %g0, [%o0+0x28]%asi 394 stxa %g0, [%o0+0x20]%asi 395 stxa %g0, [%o0+0x18]%asi 396 stxa %g0, [%o0+0x10]%asi 397 stxa %g0, [%o0+0x08]%asi 398 stxa %g0, [%o0]%asi 399 .zinst: 400 add %o0, %o3, %o0 ! increment source address 401 sub %o1, %o3, %o1 ! decrement count 402 .bzero_nobuf: 403 cmp %o1, 0x100 ! can we do whole chunk? 404 bgeu,a %ncc, .bzero_blk 405 stxa %g0, [%o0+0xf8]%asi ! do first double of chunk 406 407 cmp %o1, 7 ! can we zero any more double words 408 bleu %ncc, .byteclr ! too small go zero bytes 409 410 andn %o1, 7, %o3 ! %o3 bytes left, double-word aligned 411 srl %o3, 1, %o2 ! using doubles, need 1 instr / 2 words 412 set .zinst, %o4 ! address of clr instructions 413 sub %o4, %o2, %o4 ! jmp address relative to instr 414 jmp %o4 415 nop 416 ! 417 ! do leftover bytes 418 ! 419 3: 420 add %o0, 1, %o0 ! increment address 421 .byteclr: 422 subcc %o1, 1, %o1 ! decrement count 423 bgeu,a %ncc, 3b 424 stba %g0, [%o0]%asi ! zero a byte 425 426 .bzero_finished: 427 ! 428 ! We're just concerned with whether t_lofault was set 429 ! when we came in. We end up here from either kzero() 430 ! or bzero(). kzero() *always* sets a lofault handler. 431 ! It ors LOFAULT_SET into %o5 to indicate it has done 432 ! this even if the value of %o5 is otherwise zero. 433 ! bzero() sets a lofault handler *only* if one was 434 ! previously set. Accordingly we need to examine 435 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET 436 ! before resetting the error handler. 437 ! 438 tst %o5 439 bz %ncc, 1f 440 andn %o5, LOFAULT_SET, %o5 441 membar #Sync ! sync error barrier 442 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 443 1: 444 retl 445 clr %o0 ! return (0) 446 447 SET_SIZE(bzero)