1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #pragma ident "%Z%%M% %I% %E% SMI"
28
29 #include <sys/param.h>
30 #include <sys/errno.h>
31 #include <sys/asm_linkage.h>
32 #include <sys/vtrace.h>
33 #include <sys/machthread.h>
34 #include <sys/clock.h>
35 #include <sys/asi.h>
36 #include <sys/fsr.h>
37 #include <sys/privregs.h>
38
39 #if !defined(lint)
40 #include "assym.h"
41 #endif /* lint */
42
43 #define FP_USED 1
44 #define LOFAULT_SET 2
45
46 /*
47 * Error barrier:
48 * We use membar sync to establish an error barrier for
49 * deferred errors. Membar syncs are added before any update
50 * to t_lofault to ensure that deferred errors from earlier
51 * accesses will not be reported after the membar. This error
52 * isolation is important when we try to recover from async
53 * errors which tries to distinguish kernel accesses to user
54 * data.
55 */
56
57 /*
58 * Zero a block of storage.
59 *
60 * uzero is used by the kernel to zero a block in user address space.
61 */
62
63 #if defined(lint)
64
65 /* ARGSUSED */
66 int
67 kzero(void *addr, size_t count)
68 { return(0); }
69
70 /* ARGSUSED */
71 void
72 uzero(void *addr, size_t count)
73 {}
74
75 #else /* lint */
76
77 ENTRY(uzero)
78 !
79 ! Set a new lo_fault handler only if we came in with one
80 ! already specified.
81 !
82 wr %g0, ASI_USER, %asi
83 ldn [THREAD_REG + T_LOFAULT], %o5
84 tst %o5
85 bz,pt %ncc, .do_zero
86 sethi %hi(.zeroerr), %o2
87 or %o2, %lo(.zeroerr), %o2
88 membar #Sync
89 ba,pt %ncc, .do_zero
90 stn %o2, [THREAD_REG + T_LOFAULT]
91
92 ENTRY(kzero)
93 !
94 ! Always set a lo_fault handler
95 !
96 wr %g0, ASI_P, %asi
97 ldn [THREAD_REG + T_LOFAULT], %o5
98 sethi %hi(.zeroerr), %o2
99 or %o5, LOFAULT_SET, %o5
100 or %o2, %lo(.zeroerr), %o2
101 membar #Sync
102 ba,pt %ncc, .do_zero
103 stn %o2, [THREAD_REG + T_LOFAULT]
104
105 /*
106 * We got here because of a fault during kzero or if
107 * uzero or bzero was called with t_lofault non-zero.
108 * Otherwise we've already run screaming from the room.
109 * Errno value is in %g1. Note that we're here iff
110 * we did set t_lofault.
111 */
112 .zeroerr:
113 !
114 ! Undo asi register setting. Just set it to be the
115 ! kernel default without checking.
116 !
117 wr %g0, ASI_P, %asi
118 !
119 ! If saved t_lofault has FP_USED set, clear the %fprs register
120 !
121 btst FP_USED, %o5
122 bz,pt %ncc, 1f ! skip if not used
123 nop
124 membar #Sync
125 wr %g0, %g0, %fprs ! clear fprs
126 andn %o5, FP_USED, %o5 ! turn off flag bit
127 !
128 ! We did set t_lofault. It may well have been zero coming in.
129 !
130 1:
131 tst %o5
132 membar #Sync
133 bne,pn %ncc, 3f
134 andncc %o5, LOFAULT_SET, %o5
135 2:
136 !
137 ! Old handler was zero. Just return the error.
138 !
139 retl ! return
140 mov %g1, %o0 ! error code from %g1
141 3:
142 !
143 ! We're here because %o5 was non-zero. It was non-zero
144 ! because either LOFAULT_SET was present, a previous fault
145 ! handler was present or both. In all cases we need to reset
146 ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
147 ! before we either simply return the error or we invoke the
148 ! previously specified handler.
149 !
150 be %ncc, 2b
151 stn %o5, [THREAD_REG + T_LOFAULT]
152 jmp %o5 ! goto real handler
153 nop
154 SET_SIZE(kzero)
155 SET_SIZE(uzero)
156
157 #endif /* lint */
158
159 /*
160 * Zero a block of storage.
161 */
162
163 #if defined(lint)
164
165 /* ARGSUSED */
166 void
167 bzero(void *addr, size_t count)
168 {}
169
170 #else /* lint */
171
172 ENTRY(bzero)
173 wr %g0, ASI_P, %asi
174
175 ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector
176 tst %o5
177 bz,pt %ncc, .do_zero
178 sethi %hi(.zeroerr), %o2
179 or %o2, %lo(.zeroerr), %o2
180 membar #Sync ! sync error barrier
181 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
182
183 .do_zero:
184 cmp %o1, 15 ! check for small counts
185 blu,pn %ncc, .byteclr ! just clear bytes
186 nop
187
188 cmp %o1, 192 ! check for large counts
189 blu %ncc, .bzero_small
190 nop
191
192 sethi %hi(use_hw_bzero), %o2
193 ld [%o2 + %lo(use_hw_bzero)], %o2
194 tst %o2
195 bz %icc, .bzero_small
196 nop
197
198 rd %fprs, %o2 ! check for unused fp
199 btst FPRS_FEF, %o2
200 bnz %icc, .bzero_small
201 nop
202
203 ldn [THREAD_REG + T_LWP], %o2
204 tst %o2
205 bz,pn %ncc, .bzero_small
206 nop
207
208 ! Check for block alignment
209 btst (64-1), %o0
210 bz %icc, .bzl_block
211 nop
212
213 ! Check for double-word alignment
214 btst (8-1), %o0
215 bz %icc, .bzl_dword
216 nop
217
218 ! Check for word alignment
219 btst (4-1), %o0
220 bz %icc, .bzl_word
221 nop
222
223 ! Clear bytes until word aligned
224 .bzl_byte:
225 stba %g0, [%o0]%asi
226 add %o0, 1, %o0
227 btst (4-1), %o0
228 bnz %icc, .bzl_byte
229 sub %o1, 1, %o1
230
231 ! Check for dword-aligned
232 btst (8-1), %o0
233 bz %icc, .bzl_dword
234 nop
235
236 ! Clear words until double-word aligned
237 .bzl_word:
238 sta %g0, [%o0]%asi
239 add %o0, 4, %o0
240 btst (8-1), %o0
241 bnz %icc, .bzl_word
242 sub %o1, 4, %o1
243
244 .bzl_dword:
245 ! Clear dwords until block aligned
246 stxa %g0, [%o0]%asi
247 add %o0, 8, %o0
248 btst (64-1), %o0
249 bnz %icc, .bzl_dword
250 sub %o1, 8, %o1
251
252 .bzl_block:
253 membar #StoreStore|#StoreLoad|#LoadStore
254 wr %g0, FPRS_FEF, %fprs
255
256 ! Set the lower bit in the saved t_lofault to indicate
257 ! that we need to clear the %fprs register on the way
258 ! out
259 or %o5, FP_USED, %o5
260
261 ! Clear block
262 fzero %d0
263 fzero %d2
264 fzero %d4
265 fzero %d6
266 fzero %d8
267 fzero %d10
268 fzero %d12
269 fzero %d14
270 rd %asi, %o3
271 wr %g0, ASI_BLK_P, %asi
272 cmp %o3, ASI_P
273 bne,a %icc, 1f
274 wr %g0, ASI_BLK_AIUS, %asi
275 1:
276 mov 256, %o3
277 ba,pt %ncc, .bzl_doblock
278 nop
279
280 .bzl_blkstart:
281 ! stda %d0, [%o0+192]%asi ! in dly slot of branch that got us here
282 stda %d0, [%o0+128]%asi
283 stda %d0, [%o0+64]%asi
284 stda %d0, [%o0]%asi
285 .bzl_zinst:
286 add %o0, %o3, %o0
287 sub %o1, %o3, %o1
288 .bzl_doblock:
289 cmp %o1, 256
290 bgeu,a %ncc, .bzl_blkstart
291 stda %d0, [%o0+192]%asi
292
293 cmp %o1, 64
294 blu %ncc, .bzl_finish
295
296 andn %o1, (64-1), %o3
297 srl %o3, 4, %o2 ! using blocks, 1 instr / 16 words
298 set .bzl_zinst, %o4
299 sub %o4, %o2, %o4
300 jmp %o4
301 nop
302
303 .bzl_finish:
304 membar #StoreLoad|#StoreStore
305 wr %g0, %g0, %fprs
306 andn %o5, FP_USED, %o5
307
308 rd %asi, %o4
309 wr %g0, ASI_P, %asi
310 cmp %o4, ASI_BLK_P
311 bne,a %icc, 1f
312 wr %g0, ASI_USER, %asi
313 1:
314
315 .bzlf_dword:
316 ! double words
317 cmp %o1, 8
318 blu %ncc, .bzlf_word
319 nop
320 stxa %g0, [%o0]%asi
321 add %o0, 8, %o0
322 sub %o1, 8, %o1
323 ba,pt %ncc, .bzlf_dword
324 nop
325
326 .bzlf_word:
327 ! words
328 cmp %o1, 4
329 blu %ncc, .bzlf_byte
330 nop
331 sta %g0, [%o0]%asi
332 add %o0, 4, %o0
333 sub %o1, 4, %o1
334 ba,pt %ncc, .bzlf_word
335 nop
336
337 1:
338 add %o0, 1, %o0 ! increment address
339 .bzlf_byte:
340 subcc %o1, 1, %o1 ! decrement count
341 bgeu,a %ncc, 1b
342 stba %g0, [%o0]%asi ! zero a byte
343
344 !
345 ! If we used the FP registers, that bit was turned
346 ! off after we were finished. We're just concerned with
347 ! whether t_lofault was set when we came in. We end up
348 ! here from either kzero() or bzero(). kzero() *always*
349 ! sets a lofault handler. It ors LOFAULT_SET into %o5
350 ! to indicate it has done this even if the value of %o5
351 ! is otherwise zero. bzero() sets a lofault handler *only*
352 ! if one was previously set. Accordingly we need to examine
353 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET
354 ! before resetting the error handler.
355 !
356 tst %o5
357 bz,pt %ncc, 1f
358 andn %o5, LOFAULT_SET, %o5
359 membar #Sync ! sync error barrier
360 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
361 1:
362 retl
363 clr %o0 ! return (0)
364
365 .bzero_small:
366
367 !
368 ! Check for word alignment.
369 !
370 btst 3, %o0
371 bz .bzero_probe
372 mov 0x100, %o3 ! constant size of main loop
373 !
374 !
375 ! clear bytes until word aligned
376 !
377 1: stba %g0,[%o0]%asi
378 add %o0, 1, %o0
379 btst 3, %o0
380 bnz 1b
381 sub %o1, 1, %o1
382 .bzero_probe:
383
384 !
385 ! if needed move a word to become double-word aligned.
386 !
387 btst 7, %o0 ! is double aligned?
388 bz %icc, .bzero_nobuf
389 nop
390 sta %g0, [%o0]%asi ! clr to double boundry
391 sub %o1, 4, %o1
392 ba,pt %ncc, .bzero_nobuf
393 add %o0, 4, %o0
394
395 !stxa %g0, [%o0+0xf8]%asi
396 .bzero_blk:
397 stxa %g0, [%o0+0xf0]%asi
398 stxa %g0, [%o0+0xe8]%asi
399 stxa %g0, [%o0+0xe0]%asi
400 stxa %g0, [%o0+0xd8]%asi
401 stxa %g0, [%o0+0xd0]%asi
402 stxa %g0, [%o0+0xc8]%asi
403 stxa %g0, [%o0+0xc0]%asi
404 stxa %g0, [%o0+0xb8]%asi
405 stxa %g0, [%o0+0xb0]%asi
406 stxa %g0, [%o0+0xa8]%asi
407 stxa %g0, [%o0+0xa0]%asi
408 stxa %g0, [%o0+0x98]%asi
409 stxa %g0, [%o0+0x90]%asi
410 stxa %g0, [%o0+0x88]%asi
411 stxa %g0, [%o0+0x80]%asi
412 stxa %g0, [%o0+0x78]%asi
413 stxa %g0, [%o0+0x70]%asi
414 stxa %g0, [%o0+0x68]%asi
415 stxa %g0, [%o0+0x60]%asi
416 stxa %g0, [%o0+0x58]%asi
417 stxa %g0, [%o0+0x50]%asi
418 stxa %g0, [%o0+0x48]%asi
419 stxa %g0, [%o0+0x40]%asi
420 stxa %g0, [%o0+0x38]%asi
421 stxa %g0, [%o0+0x30]%asi
422 stxa %g0, [%o0+0x28]%asi
423 stxa %g0, [%o0+0x20]%asi
424 stxa %g0, [%o0+0x18]%asi
425 stxa %g0, [%o0+0x10]%asi
426 stxa %g0, [%o0+0x08]%asi
427 stxa %g0, [%o0]%asi
428 .zinst:
429 add %o0, %o3, %o0 ! increment source address
430 sub %o1, %o3, %o1 ! decrement count
431 .bzero_nobuf:
432 cmp %o1, 0x100 ! can we do whole chunk?
433 bgeu,a %ncc, .bzero_blk
434 stxa %g0, [%o0+0xf8]%asi ! do first double of chunk
435
436 cmp %o1, 7 ! can we zero any more double words
437 bleu %ncc, .byteclr ! too small go zero bytes
438
439 andn %o1, 7, %o3 ! %o3 bytes left, double-word aligned
440 srl %o3, 1, %o2 ! using doubles, need 1 instr / 2 words
441 set .zinst, %o4 ! address of clr instructions
442 sub %o4, %o2, %o4 ! jmp address relative to instr
443 jmp %o4
444 nop
445 !
446 ! do leftover bytes
447 !
448 3:
449 add %o0, 1, %o0 ! increment address
450 .byteclr:
451 subcc %o1, 1, %o1 ! decrement count
452 bgeu,a %ncc, 3b
453 stba %g0, [%o0]%asi ! zero a byte
454
455 .bzero_finished:
456 !
457 ! We're just concerned with whether t_lofault was set
458 ! when we came in. We end up here from either kzero()
459 ! or bzero(). kzero() *always* sets a lofault handler.
460 ! It ors LOFAULT_SET into %o5 to indicate it has done
461 ! this even if the value of %o5 is otherwise zero.
462 ! bzero() sets a lofault handler *only* if one was
463 ! previously set. Accordingly we need to examine
464 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET
465 ! before resetting the error handler.
466 !
467 tst %o5
468 bz %ncc, 1f
469 andn %o5, LOFAULT_SET, %o5
470 membar #Sync ! sync error barrier
471 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
472 1:
473 retl
474 clr %o0 ! return (0)
475
476 SET_SIZE(bzero)
477 #endif /* lint */