1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/param.h>
27 #include <sys/errno.h>
28 #include <sys/asm_linkage.h>
29 #include <sys/vtrace.h>
30 #include <sys/machthread.h>
31 #include <sys/clock.h>
32 #include <sys/asi.h>
33 #include <sys/fsr.h>
34 #include <sys/privregs.h>
35
36 #include "assym.h"
37
38 /*
39 * Error barrier:
40 * We use membar sync to establish an error barrier for
41 * deferred errors. Membar syncs are added before any update
42 * to t_lofault to ensure that deferred errors from earlier
43 * accesses will not be reported after the membar. This error
44 * isolation is important when we try to recover from async
45 * errors which tries to distinguish kernel accesses to user
46 * data.
47 */
48
49 /*
50 * Copy a null terminated string from one point to another in
51 * the kernel address space.
52 * NOTE - don't use %o5 in this routine as copy{in,out}str uses it.
53 *
54 * copystr(from, to, maxlength, lencopied)
55 * caddr_t from, to;
56 * u_int maxlength, *lencopied;
57 */
58
59 ENTRY(copystr)
60 orcc %o2, %g0, %o4 ! save original count
61 bg,a %ncc, 1f
62 sub %o0, %o1, %o0 ! o0 gets the difference of src and dst
63
64 !
65 ! maxlength <= 0
66 !
67 bz %ncc, .cs_out ! maxlength = 0
68 mov ENAMETOOLONG, %o0
69
70 b 2f ! maxlength < 0
71 mov EFAULT, %o0 ! return failure
72
73 !
74 ! Do a byte by byte loop.
75 ! We do this instead of a word by word copy because most strings
76 ! are small and this takes a small number of cache lines.
77 !
78 0:
79 stb %g1, [%o1] ! store byte
80 tst %g1
81 bnz,pt %icc, 1f
82 add %o1, 1, %o1 ! incr dst addr
83
84 ba,pt %ncc, .cs_out ! last byte in string
85 mov 0, %o0 ! ret code = 0
86 1:
87 subcc %o2, 1, %o2 ! test count
88 bgeu,a %ncc, 0b
89 ldub [%o0 + %o1], %g1 ! delay slot, get source byte
90
91 mov 0, %o2 ! max number of bytes moved
92 mov ENAMETOOLONG, %o0 ! ret code = ENAMETOOLONG
93 .cs_out:
94 tst %o3
95 bz %ncc, 2f
96 sub %o4, %o2, %o4 ! compute length and store it
97 stn %o4, [%o3]
98 2:
99 retl
100 nop
101 SET_SIZE(copystr)
102
103
104 /*
105 * Copy a null terminated string from the user address space into
106 * the kernel address space.
107 */
108
109 ENTRY(copyinstr)
110 sethi %hi(.copyinstr_err), %o4
111 ldn [THREAD_REG + T_LOFAULT], %o5 ! catch faults
112 or %o4, %lo(.copyinstr_err), %o4
113 membar #Sync ! sync error barrier
114 stn %o4, [THREAD_REG + T_LOFAULT]
115
116 brz,a,pn %o2, .copyinstr_out
117 mov ENAMETOOLONG, %o0
118
119 mov %o2, %g3 ! g3 is the current count
120 mov %o1, %g4 ! g4 is the dest addr
121
122 b 1f
123 sub %o0, %o1, %g2 ! g2 gets the difference of src and dst
124
125 !
126 ! Do a byte by byte loop.
127 ! We do this instead of a word by word copy because most strings
128 ! are small and this takes a small number of cache lines.
129 !
130 0:
131 stb %g1, [%g4] ! store byte
132 tst %g1
133 bnz,pt %icc, 1f
134 add %g4, 1, %g4 ! incr dst addr
135
136 ba,pt %ncc, .copyinstr_out ! last byte in string
137 mov 0, %o0 ! ret code = 0
138 1:
139 subcc %g3, 1, %g3 ! test count
140 bgeu,a %ncc, 0b
141 lduba [%g2+%g4]ASI_USER, %g1 ! delay slot, get source byte
142
143 mov 0, %g3 ! max number of bytes moved
144 ba,pt %ncc, .copyinstr_out
145 mov ENAMETOOLONG, %o0 ! ret code = ENAMETOOLONG
146
147 /*
148 * Fault while trying to move from or to user space.
149 * Set and return error code.
150 */
151 .copyinstr_err:
152 membar #Sync ! sync error barrier
153 stn %o5, [THREAD_REG + T_LOFAULT]
154 ldn [THREAD_REG + T_COPYOPS], %o4
155 brz %o4, 1f
156 nop
157 ldn [%o4 + CP_COPYINSTR], %g1
158 jmp %g1
159 nop
160 1:
161 retl
162 mov EFAULT, %o0
163 .copyinstr_out:
164 tst %o3 ! want length?
165 bz %ncc, 2f
166 sub %o2, %g3, %o2 ! compute length and store it
167 stn %o2, [%o3]
168 2:
169 membar #Sync ! sync error barrier
170 retl
171 stn %o5, [THREAD_REG + T_LOFAULT] ! stop catching faults
172 SET_SIZE(copyinstr)
173
174 ENTRY(copyinstr_noerr)
175 mov %o2, %o4 ! save original count
176
177 ! maxlength is unsigned so the only error is if it's 0
178 brz,a,pn %o2, .copyinstr_noerr_out
179 mov ENAMETOOLONG, %o0
180
181 b 1f
182 sub %o0, %o1, %o0 ! o0 gets the difference of src and dst
183
184 !
185 ! Do a byte by byte loop.
186 ! We do this instead of a word by word copy because most strings
187 ! are small and this takes a small number of cache lines.
188 !
189 0:
190 stb %g1, [%o1] ! store byte
191 tst %g1 ! null byte?
192 bnz 1f
193 add %o1, 1, %o1 ! incr dst addr
194
195 ba,pt %ncc, .copyinstr_noerr_out ! last byte in string
196 mov 0, %o0 ! ret code = 0
197 1:
198 subcc %o2, 1, %o2 ! test count
199 bgeu,a %ncc, 0b
200 lduba [%o0 + %o1]ASI_USER, %g1 ! delay slot, get source byte
201
202 mov 0, %o2 ! max number of bytes moved
203 b .copyinstr_noerr_out
204 mov ENAMETOOLONG, %o0 ! ret code = ENAMETOOLONG
205 .copyinstr_noerr_out:
206 tst %o3 ! want length?
207 bz %ncc, 2f
208 sub %o4, %o2, %o4
209 stn %o4, [%o3]
210 2:
211 retl
212 nop
213 SET_SIZE(copyinstr_noerr)
214
215 /*
216 * Copy a null terminated string from the kernel
217 * address space to the user address space.
218 */
219
220 ENTRY(copyoutstr)
221 sethi %hi(.copyoutstr_err), %o5
222 ldn [THREAD_REG + T_LOFAULT], %o4 ! catch faults
223 or %o5, %lo(.copyoutstr_err), %o5
224 membar #Sync ! sync error barrier
225 stn %o5, [THREAD_REG + T_LOFAULT]
226 mov %o4, %o5
227
228 brz,a,pn %o2, .copyoutstr_out
229 mov ENAMETOOLONG, %o0
230
231 mov %o2, %g3 ! g3 is the current count
232 mov %o1, %g4 ! g4 is the dest addr
233
234 b 1f
235 sub %o0, %o1, %g2 ! g2 gets the difference of src and dst
236
237 !
238 ! Do a byte by byte loop.
239 ! We do this instead of a word by word copy because most strings
240 ! are small and this takes a small number of cache lines.
241 !
242 0:
243 stba %g1, [%g4]ASI_USER ! store byte
244 tst %g1
245 bnz,pt %icc, 1f
246 add %g4, 1, %g4 ! incr dst addr
247
248 ba,pt %ncc, .copyoutstr_out ! last byte in string
249 mov 0, %o0 ! ret code = 0
250 1:
251 subcc %g3, 1, %g3 ! test count
252 bgeu,a %ncc, 0b
253 ldub [%g2 + %g4], %g1 ! delay slot, get source byte
254
255 mov 0, %g3 ! max number of bytes moved
256 ba,pt %ncc, .copyoutstr_out
257 mov ENAMETOOLONG, %o0 ! ret code = ENAMETOOLONG
258
259 /*
260 * Fault while trying to move from or to user space.
261 * Set and return error code.
262 */
263 .copyoutstr_err:
264 membar #Sync ! sync error barrier
265 stn %o5, [THREAD_REG + T_LOFAULT]
266 ldn [THREAD_REG + T_COPYOPS], %o4
267 brz %o4, 1f
268 nop
269 ldn [%o4 + CP_COPYOUTSTR], %g1
270 jmp %g1
271 nop
272 1:
273 retl
274 mov EFAULT, %o0
275 .copyoutstr_out:
276 tst %o3 ! want length?
277 bz %ncc, 2f
278 sub %o2, %g3, %o2 ! compute length and store it
279 stn %o2, [%o3]
280 2:
281 membar #Sync ! sync error barrier
282 retl
283 stn %o5, [THREAD_REG + T_LOFAULT] ! stop catching faults
284 SET_SIZE(copyoutstr)
285
286 ENTRY(copyoutstr_noerr)
287 mov %o2, %o4 ! save original count
288
289 brz,a,pn %o2, .copyoutstr_noerr_out
290 mov ENAMETOOLONG, %o0
291
292 b 1f
293 sub %o0, %o1, %o0 ! o0 gets the difference of src and dst
294
295 !
296 ! Do a byte by byte loop.
297 ! We do this instead of a word by word copy because most strings
298 ! are small and this takes a small number of cache lines.
299 !
300 0:
301 stba %g1, [%o1]ASI_USER ! store byte
302 tst %g1 ! null byte?
303 bnz 1f
304 add %o1, 1, %o1 ! incr dst addr
305
306 b .copyoutstr_noerr_out ! last byte in string
307 mov 0, %o0 ! ret code = 0
308 1:
309 subcc %o2, 1, %o2 ! test count
310 bgeu,a %ncc, 0b
311 ldub [%o0+%o1], %g1 ! delay slot, get source byte
312
313 mov 0, %o2 ! max number of bytes moved
314 b .copyoutstr_noerr_out
315 mov ENAMETOOLONG, %o0 ! ret code = ENAMETOOLONG
316 .copyoutstr_noerr_out:
317 tst %o3 ! want length?
318 bz %ncc, 2f
319 sub %o4, %o2, %o4
320 stn %o4, [%o3]
321 2:
322 retl
323 nop
324 SET_SIZE(copyoutstr_noerr)
325
326
327 /*
328 * Copy a block of storage. If the source and target regions overlap,
329 * one or both of the regions will be silently corrupted.
330 * No fault handler installed (to be called under on_fault())
331 */
332
333 ENTRY(ucopy)
334 save %sp, -SA(MINFRAME), %sp ! get another window
335
336 subcc %g0, %i2, %i3
337 add %i0, %i2, %i0
338 bz,pn %ncc, 5f
339 add %i1, %i2, %i1
340 lduba [%i0 + %i3]ASI_USER, %i4
341 4: stba %i4, [%i1 + %i3]ASI_USER
342 inccc %i3
343 bcc,a,pt %ncc, 4b
344 lduba [%i0 + %i3]ASI_USER, %i4
345 5:
346 ret
347 restore %g0, 0, %o0 ! return (0)
348
349 SET_SIZE(ucopy)
350
351 /*
352 * Copy a user-land string. If the source and target regions overlap,
353 * one or both of the regions will be silently corrupted.
354 * No fault handler installed (to be called under on_fault())
355 */
356
357 ENTRY(ucopystr)
358 save %sp, -SA(MINFRAME), %sp ! get another window
359
360 brz %i2, 5f
361 clr %i5
362
363 lduba [%i0 + %i5]ASI_USER, %i4
364 4: stba %i4, [%i1 + %i5]ASI_USER
365 brz,pn %i4, 5f
366 inc %i5
367 deccc %i2
368 bnz,a,pt %ncc, 4b
369 lduba [%i0 + %i5]ASI_USER, %i4
370 5:
371 brnz,a,pt %i3, 6f
372 stn %i5, [%i3]
373 6:
374 ret
375 restore %g0, 0, %o0 ! return (0)
376
377 SET_SIZE(ucopystr)