Print this page
de-linting of .s files
first
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/copy.s
+++ new/usr/src/uts/intel/ia32/ml/copy.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /*
27 27 * Copyright (c) 2009, Intel Corporation
28 28 * All rights reserved.
29 29 */
30 30
31 31 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
32 32 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
33 33 /* All Rights Reserved */
34 34
|
↓ open down ↓ |
34 lines elided |
↑ open up ↑ |
35 35 /* Copyright (c) 1987, 1988 Microsoft Corporation */
36 36 /* All Rights Reserved */
37 37
38 38 /*
39 39 * Copyright 2019 Joyent, Inc.
40 40 */
41 41
42 42 #include <sys/errno.h>
43 43 #include <sys/asm_linkage.h>
44 44
45 -#if defined(__lint)
46 -#include <sys/types.h>
47 -#include <sys/systm.h>
48 -#else /* __lint */
49 45 #include "assym.h"
50 -#endif /* __lint */
51 46
52 47 #define KCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
53 48 #define XCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
54 49 /*
55 50 * Non-temopral access (NTA) alignment requirement
56 51 */
57 52 #define NTA_ALIGN_SIZE 4 /* Must be at least 4-byte aligned */
58 53 #define NTA_ALIGN_MASK _CONST(NTA_ALIGN_SIZE-1)
59 54 #define COUNT_ALIGN_SIZE 16 /* Must be at least 16-byte aligned */
60 55 #define COUNT_ALIGN_MASK _CONST(COUNT_ALIGN_SIZE-1)
61 56
62 57 /*
63 58 * With the introduction of Broadwell, Intel has introduced supervisor mode
64 59 * access protection -- SMAP. SMAP forces the kernel to set certain bits to
65 60 * enable access of user pages (AC in rflags, defines as PS_ACHK in
66 61 * <sys/psw.h>). One of the challenges is that the implementation of many of the
67 62 * userland copy routines directly use the kernel ones. For example, copyin and
68 63 * copyout simply go and jump to the do_copy_fault label and traditionally let
69 64 * those deal with the return for them. In fact, changing that is a can of frame
70 65 * pointers.
71 66 *
72 67 * Rules and Constraints:
73 68 *
74 69 * 1. For anything that's not in copy.s, we have it do explicit calls to the
75 70 * smap related code. It usually is in a position where it is able to. This is
76 71 * restricted to the following three places: DTrace, resume() in swtch.s and
77 72 * on_fault/no_fault. If you want to add it somewhere else, we should be
78 73 * thinking twice.
79 74 *
80 75 * 2. We try to toggle this at the smallest window possible. This means that if
81 76 * we take a fault, need to try to use a copyop in copyin() or copyout(), or any
82 77 * other function, we will always leave with SMAP enabled (the kernel cannot
83 78 * access user pages).
84 79 *
85 80 * 3. None of the *_noerr() or ucopy/uzero routines should toggle SMAP. They are
86 81 * explicitly only allowed to be called while in an on_fault()/no_fault() handler,
87 82 * which already takes care of ensuring that SMAP is enabled and disabled. Note
88 83 * this means that when under an on_fault()/no_fault() handler, one must not
89 84 * call the non-*_noeer() routines.
90 85 *
91 86 * 4. The first thing we should do after coming out of an lofault handler is to
92 87 * make sure that we call smap_enable again to ensure that we are safely
93 88 * protected, as more often than not, we will have disabled smap to get there.
94 89 *
95 90 * 5. The SMAP functions, smap_enable and smap_disable may not touch any
96 91 * registers beyond those done by the call and ret. These routines may be called
97 92 * from arbitrary contexts in copy.s where we have slightly more special ABIs in
98 93 * place.
99 94 *
100 95 * 6. For any inline user of SMAP, the appropriate SMAP_ENABLE_INSTR and
101 96 * SMAP_DISABLE_INSTR macro should be used (except for smap_enable() and
102 97 * smap_disable()). If the number of these is changed, you must update the
103 98 * constants SMAP_ENABLE_COUNT and SMAP_DISABLE_COUNT below.
104 99 *
105 100 * 7. Note, at this time SMAP is not implemented for the 32-bit kernel. There is
106 101 * no known technical reason preventing it from being enabled.
107 102 *
108 103 * 8. Generally this .s file is processed by a K&R style cpp. This means that it
109 104 * really has a lot of feelings about whitespace. In particular, if you have a
110 105 * macro FOO with the arguments FOO(1, 3), the second argument is in fact ' 3'.
111 106 *
112 107 * 9. The smap_enable and smap_disable functions should not generally be called.
113 108 * They exist such that DTrace and on_trap() may use them, that's it.
114 109 *
115 110 * 10. In general, the kernel has its own value for rflags that gets used. This
116 111 * is maintained in a few different places which vary based on how the thread
117 112 * comes into existence and whether it's a user thread. In general, when the
118 113 * kernel takes a trap, it always will set ourselves to a known set of flags,
119 114 * mainly as part of ENABLE_INTR_FLAGS and F_OFF and F_ON. These ensure that
120 115 * PS_ACHK is cleared for us. In addition, when using the sysenter instruction,
121 116 * we mask off PS_ACHK off via the AMD_SFMASK MSR. See init_cpu_syscall() for
122 117 * where that gets masked off.
123 118 */
124 119
125 120 /*
126 121 * The optimal 64-bit bcopy and kcopy for modern x86 processors uses
127 122 * "rep smovq" for large sizes. Performance data shows that many calls to
128 123 * bcopy/kcopy/bzero/kzero operate on small buffers. For best performance for
129 124 * these small sizes unrolled code is used. For medium sizes loops writing
130 125 * 64-bytes per loop are used. Transition points were determined experimentally.
131 126 */
132 127 #define BZERO_USE_REP (1024)
133 128 #define BCOPY_DFLT_REP (128)
134 129 #define BCOPY_NHM_REP (768)
135 130
|
↓ open down ↓ |
75 lines elided |
↑ open up ↑ |
136 131 /*
137 132 * Copy a block of storage, returning an error code if `from' or
138 133 * `to' takes a kernel pagefault which cannot be resolved.
139 134 * Returns errno value on pagefault error, 0 if all ok
140 135 */
141 136
142 137 /*
143 138 * I'm sorry about these macros, but copy.s is unsurprisingly sensitive to
144 139 * additional call instructions.
145 140 */
146 -#if defined(__amd64)
147 141 #define SMAP_DISABLE_COUNT 16
148 142 #define SMAP_ENABLE_COUNT 26
149 -#elif defined(__i386)
150 -#define SMAP_DISABLE_COUNT 0
151 -#define SMAP_ENABLE_COUNT 0
152 -#endif
153 143
154 144 #define SMAP_DISABLE_INSTR(ITER) \
155 145 .globl _smap_disable_patch_/**/ITER; \
156 146 _smap_disable_patch_/**/ITER/**/:; \
157 147 nop; nop; nop;
158 148
159 149 #define SMAP_ENABLE_INSTR(ITER) \
160 150 .globl _smap_enable_patch_/**/ITER; \
161 151 _smap_enable_patch_/**/ITER/**/:; \
162 152 nop; nop; nop;
163 153
164 -#if defined(__lint)
165 -
166 -/* ARGSUSED */
167 -int
168 -kcopy(const void *from, void *to, size_t count)
169 -{ return (0); }
170 -
171 -#else /* __lint */
172 -
173 154 .globl kernelbase
174 155 .globl postbootkernelbase
175 156
176 -#if defined(__amd64)
177 -
178 157 ENTRY(kcopy)
179 158 pushq %rbp
180 159 movq %rsp, %rbp
181 160 #ifdef DEBUG
182 161 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
183 162 jb 0f
184 163 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
185 164 jnb 1f
186 165 0: leaq .kcopy_panic_msg(%rip), %rdi
187 166 xorl %eax, %eax
188 167 call panic
189 168 1:
190 169 #endif
191 170 /*
192 171 * pass lofault value as 4th argument to do_copy_fault
193 172 */
194 173 leaq _kcopy_copyerr(%rip), %rcx
195 174 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
196 175
197 176 do_copy_fault:
198 177 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
199 178 movq %rcx, T_LOFAULT(%r9) /* new lofault */
200 179 call bcopy_altentry
201 180 xorl %eax, %eax /* return 0 (success) */
202 181 SMAP_ENABLE_INSTR(0)
203 182
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
204 183 /*
205 184 * A fault during do_copy_fault is indicated through an errno value
206 185 * in %rax and we iretq from the trap handler to here.
207 186 */
208 187 _kcopy_copyerr:
209 188 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
210 189 leave
211 190 ret
212 191 SET_SIZE(kcopy)
213 192
214 -#elif defined(__i386)
215 -
216 -#define ARG_FROM 8
217 -#define ARG_TO 12
218 -#define ARG_COUNT 16
219 -
220 - ENTRY(kcopy)
221 -#ifdef DEBUG
222 - pushl %ebp
223 - movl %esp, %ebp
224 - movl postbootkernelbase, %eax
225 - cmpl %eax, ARG_FROM(%ebp)
226 - jb 0f
227 - cmpl %eax, ARG_TO(%ebp)
228 - jnb 1f
229 -0: pushl $.kcopy_panic_msg
230 - call panic
231 -1: popl %ebp
232 -#endif
233 - lea _kcopy_copyerr, %eax /* lofault value */
234 - movl %gs:CPU_THREAD, %edx
235 -
236 -do_copy_fault:
237 - pushl %ebp
238 - movl %esp, %ebp /* setup stack frame */
239 - pushl %esi
240 - pushl %edi /* save registers */
241 -
242 - movl T_LOFAULT(%edx), %edi
243 - pushl %edi /* save the current lofault */
244 - movl %eax, T_LOFAULT(%edx) /* new lofault */
245 -
246 - movl ARG_COUNT(%ebp), %ecx
247 - movl ARG_FROM(%ebp), %esi
248 - movl ARG_TO(%ebp), %edi
249 - shrl $2, %ecx /* word count */
250 - rep
251 - smovl
252 - movl ARG_COUNT(%ebp), %ecx
253 - andl $3, %ecx /* bytes left over */
254 - rep
255 - smovb
256 - xorl %eax, %eax
257 -
258 - /*
259 - * A fault during do_copy_fault is indicated through an errno value
260 - * in %eax and we iret from the trap handler to here.
261 - */
262 -_kcopy_copyerr:
263 - popl %ecx
264 - popl %edi
265 - movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */
266 - popl %esi
267 - popl %ebp
268 - ret
269 - SET_SIZE(kcopy)
270 -
271 193 #undef ARG_FROM
272 194 #undef ARG_TO
273 195 #undef ARG_COUNT
274 196
275 -#endif /* __i386 */
276 -#endif /* __lint */
277 -
278 -#if defined(__lint)
279 -
280 -/*
281 - * Copy a block of storage. Similar to kcopy but uses non-temporal
282 - * instructions.
283 - */
284 -
285 -/* ARGSUSED */
286 -int
287 -kcopy_nta(const void *from, void *to, size_t count, int copy_cached)
288 -{ return (0); }
289 -
290 -#else /* __lint */
291 -
292 -#if defined(__amd64)
293 -
294 197 #define COPY_LOOP_INIT(src, dst, cnt) \
295 198 addq cnt, src; \
296 199 addq cnt, dst; \
297 200 shrq $3, cnt; \
298 201 neg cnt
299 202
300 203 /* Copy 16 bytes per loop. Uses %rax and %r8 */
301 204 #define COPY_LOOP_BODY(src, dst, cnt) \
302 205 prefetchnta 0x100(src, cnt, 8); \
303 206 movq (src, cnt, 8), %rax; \
304 207 movq 0x8(src, cnt, 8), %r8; \
305 208 movnti %rax, (dst, cnt, 8); \
306 209 movnti %r8, 0x8(dst, cnt, 8); \
307 210 addq $2, cnt
308 211
309 212 ENTRY(kcopy_nta)
310 213 pushq %rbp
311 214 movq %rsp, %rbp
312 215 #ifdef DEBUG
313 216 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
314 217 jb 0f
315 218 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
316 219 jnb 1f
317 220 0: leaq .kcopy_panic_msg(%rip), %rdi
318 221 xorl %eax, %eax
319 222 call panic
320 223 1:
321 224 #endif
322 225
323 226 movq %gs:CPU_THREAD, %r9
324 227 cmpq $0, %rcx /* No non-temporal access? */
325 228 /*
326 229 * pass lofault value as 4th argument to do_copy_fault
327 230 */
328 231 leaq _kcopy_nta_copyerr(%rip), %rcx /* doesn't set rflags */
329 232 jnz do_copy_fault /* use regular access */
330 233 /*
331 234 * Make sure cnt is >= KCOPY_MIN_SIZE
332 235 */
333 236 cmpq $KCOPY_MIN_SIZE, %rdx
334 237 jb do_copy_fault
335 238
336 239 /*
337 240 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
338 241 * count is COUNT_ALIGN_SIZE aligned.
339 242 */
340 243 movq %rdi, %r10
341 244 orq %rsi, %r10
342 245 andq $NTA_ALIGN_MASK, %r10
343 246 orq %rdx, %r10
344 247 andq $COUNT_ALIGN_MASK, %r10
345 248 jnz do_copy_fault
346 249
347 250 ALTENTRY(do_copy_fault_nta)
348 251 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
349 252 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
350 253 movq %rcx, T_LOFAULT(%r9) /* new lofault */
351 254
352 255 /*
353 256 * COPY_LOOP_BODY uses %rax and %r8
354 257 */
355 258 COPY_LOOP_INIT(%rdi, %rsi, %rdx)
356 259 2: COPY_LOOP_BODY(%rdi, %rsi, %rdx)
357 260 jnz 2b
358 261
359 262 mfence
|
↓ open down ↓ |
56 lines elided |
↑ open up ↑ |
360 263 xorl %eax, %eax /* return 0 (success) */
361 264 SMAP_ENABLE_INSTR(1)
362 265
363 266 _kcopy_nta_copyerr:
364 267 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
365 268 leave
366 269 ret
367 270 SET_SIZE(do_copy_fault_nta)
368 271 SET_SIZE(kcopy_nta)
369 272
370 -#elif defined(__i386)
371 -
372 -#define ARG_FROM 8
373 -#define ARG_TO 12
374 -#define ARG_COUNT 16
375 -
376 -#define COPY_LOOP_INIT(src, dst, cnt) \
377 - addl cnt, src; \
378 - addl cnt, dst; \
379 - shrl $3, cnt; \
380 - neg cnt
381 -
382 -#define COPY_LOOP_BODY(src, dst, cnt) \
383 - prefetchnta 0x100(src, cnt, 8); \
384 - movl (src, cnt, 8), %esi; \
385 - movnti %esi, (dst, cnt, 8); \
386 - movl 0x4(src, cnt, 8), %esi; \
387 - movnti %esi, 0x4(dst, cnt, 8); \
388 - movl 0x8(src, cnt, 8), %esi; \
389 - movnti %esi, 0x8(dst, cnt, 8); \
390 - movl 0xc(src, cnt, 8), %esi; \
391 - movnti %esi, 0xc(dst, cnt, 8); \
392 - addl $2, cnt
393 -
394 - /*
395 - * kcopy_nta is not implemented for 32-bit as no performance
396 - * improvement was shown. We simply jump directly to kcopy
397 - * and discard the 4 arguments.
398 - */
399 - ENTRY(kcopy_nta)
400 - jmp kcopy
401 -
402 - lea _kcopy_nta_copyerr, %eax /* lofault value */
403 - ALTENTRY(do_copy_fault_nta)
404 - pushl %ebp
405 - movl %esp, %ebp /* setup stack frame */
406 - pushl %esi
407 - pushl %edi
408 -
409 - movl %gs:CPU_THREAD, %edx
410 - movl T_LOFAULT(%edx), %edi
411 - pushl %edi /* save the current lofault */
412 - movl %eax, T_LOFAULT(%edx) /* new lofault */
413 -
414 - /* COPY_LOOP_BODY needs to use %esi */
415 - movl ARG_COUNT(%ebp), %ecx
416 - movl ARG_FROM(%ebp), %edi
417 - movl ARG_TO(%ebp), %eax
418 - COPY_LOOP_INIT(%edi, %eax, %ecx)
419 -1: COPY_LOOP_BODY(%edi, %eax, %ecx)
420 - jnz 1b
421 - mfence
422 -
423 - xorl %eax, %eax
424 -_kcopy_nta_copyerr:
425 - popl %ecx
426 - popl %edi
427 - movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */
428 - popl %esi
429 - leave
430 - ret
431 - SET_SIZE(do_copy_fault_nta)
432 - SET_SIZE(kcopy_nta)
433 -
434 -#undef ARG_FROM
435 -#undef ARG_TO
436 -#undef ARG_COUNT
437 -
438 -#endif /* __i386 */
439 -#endif /* __lint */
440 -
441 -#if defined(__lint)
442 -
443 -/* ARGSUSED */
444 -void
445 -bcopy(const void *from, void *to, size_t count)
446 -{}
447 -
448 -#else /* __lint */
449 -
450 -#if defined(__amd64)
451 -
452 273 ENTRY(bcopy)
453 274 #ifdef DEBUG
454 275 orq %rdx, %rdx /* %rdx = count */
455 276 jz 1f
456 277 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
457 278 jb 0f
458 279 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
459 280 jnb 1f
460 281 0: leaq .bcopy_panic_msg(%rip), %rdi
461 282 jmp call_panic /* setup stack and call panic */
462 283 1:
463 284 #endif
464 285 /*
465 286 * bcopy_altentry() is called from kcopy, i.e., do_copy_fault.
466 287 * kcopy assumes that bcopy doesn't touch %r9 and %r11. If bcopy
467 288 * uses these registers in future they must be saved and restored.
468 289 */
469 290 ALTENTRY(bcopy_altentry)
470 291 do_copy:
471 292 #define L(s) .bcopy/**/s
472 293 cmpq $0x50, %rdx /* 80 */
473 294 jae bcopy_ck_size
474 295
475 296 /*
476 297 * Performance data shows many caller's copy small buffers. So for
477 298 * best perf for these sizes unrolled code is used. Store data without
478 299 * worrying about alignment.
479 300 */
480 301 leaq L(fwdPxQx)(%rip), %r10
481 302 addq %rdx, %rdi
482 303 addq %rdx, %rsi
483 304 movslq (%r10,%rdx,4), %rcx
484 305 leaq (%rcx,%r10,1), %r10
485 306 INDIRECT_JMP_REG(r10)
486 307
487 308 .p2align 4
488 309 L(fwdPxQx):
489 310 .int L(P0Q0)-L(fwdPxQx) /* 0 */
490 311 .int L(P1Q0)-L(fwdPxQx)
491 312 .int L(P2Q0)-L(fwdPxQx)
492 313 .int L(P3Q0)-L(fwdPxQx)
493 314 .int L(P4Q0)-L(fwdPxQx)
494 315 .int L(P5Q0)-L(fwdPxQx)
495 316 .int L(P6Q0)-L(fwdPxQx)
496 317 .int L(P7Q0)-L(fwdPxQx)
497 318
498 319 .int L(P0Q1)-L(fwdPxQx) /* 8 */
499 320 .int L(P1Q1)-L(fwdPxQx)
500 321 .int L(P2Q1)-L(fwdPxQx)
501 322 .int L(P3Q1)-L(fwdPxQx)
502 323 .int L(P4Q1)-L(fwdPxQx)
503 324 .int L(P5Q1)-L(fwdPxQx)
504 325 .int L(P6Q1)-L(fwdPxQx)
505 326 .int L(P7Q1)-L(fwdPxQx)
506 327
507 328 .int L(P0Q2)-L(fwdPxQx) /* 16 */
508 329 .int L(P1Q2)-L(fwdPxQx)
509 330 .int L(P2Q2)-L(fwdPxQx)
510 331 .int L(P3Q2)-L(fwdPxQx)
511 332 .int L(P4Q2)-L(fwdPxQx)
512 333 .int L(P5Q2)-L(fwdPxQx)
513 334 .int L(P6Q2)-L(fwdPxQx)
514 335 .int L(P7Q2)-L(fwdPxQx)
515 336
516 337 .int L(P0Q3)-L(fwdPxQx) /* 24 */
517 338 .int L(P1Q3)-L(fwdPxQx)
518 339 .int L(P2Q3)-L(fwdPxQx)
519 340 .int L(P3Q3)-L(fwdPxQx)
520 341 .int L(P4Q3)-L(fwdPxQx)
521 342 .int L(P5Q3)-L(fwdPxQx)
522 343 .int L(P6Q3)-L(fwdPxQx)
523 344 .int L(P7Q3)-L(fwdPxQx)
524 345
525 346 .int L(P0Q4)-L(fwdPxQx) /* 32 */
526 347 .int L(P1Q4)-L(fwdPxQx)
527 348 .int L(P2Q4)-L(fwdPxQx)
528 349 .int L(P3Q4)-L(fwdPxQx)
529 350 .int L(P4Q4)-L(fwdPxQx)
530 351 .int L(P5Q4)-L(fwdPxQx)
531 352 .int L(P6Q4)-L(fwdPxQx)
532 353 .int L(P7Q4)-L(fwdPxQx)
533 354
534 355 .int L(P0Q5)-L(fwdPxQx) /* 40 */
535 356 .int L(P1Q5)-L(fwdPxQx)
536 357 .int L(P2Q5)-L(fwdPxQx)
537 358 .int L(P3Q5)-L(fwdPxQx)
538 359 .int L(P4Q5)-L(fwdPxQx)
539 360 .int L(P5Q5)-L(fwdPxQx)
540 361 .int L(P6Q5)-L(fwdPxQx)
541 362 .int L(P7Q5)-L(fwdPxQx)
542 363
543 364 .int L(P0Q6)-L(fwdPxQx) /* 48 */
544 365 .int L(P1Q6)-L(fwdPxQx)
545 366 .int L(P2Q6)-L(fwdPxQx)
546 367 .int L(P3Q6)-L(fwdPxQx)
547 368 .int L(P4Q6)-L(fwdPxQx)
548 369 .int L(P5Q6)-L(fwdPxQx)
549 370 .int L(P6Q6)-L(fwdPxQx)
550 371 .int L(P7Q6)-L(fwdPxQx)
551 372
552 373 .int L(P0Q7)-L(fwdPxQx) /* 56 */
553 374 .int L(P1Q7)-L(fwdPxQx)
554 375 .int L(P2Q7)-L(fwdPxQx)
555 376 .int L(P3Q7)-L(fwdPxQx)
556 377 .int L(P4Q7)-L(fwdPxQx)
557 378 .int L(P5Q7)-L(fwdPxQx)
558 379 .int L(P6Q7)-L(fwdPxQx)
559 380 .int L(P7Q7)-L(fwdPxQx)
560 381
561 382 .int L(P0Q8)-L(fwdPxQx) /* 64 */
562 383 .int L(P1Q8)-L(fwdPxQx)
563 384 .int L(P2Q8)-L(fwdPxQx)
564 385 .int L(P3Q8)-L(fwdPxQx)
565 386 .int L(P4Q8)-L(fwdPxQx)
566 387 .int L(P5Q8)-L(fwdPxQx)
567 388 .int L(P6Q8)-L(fwdPxQx)
568 389 .int L(P7Q8)-L(fwdPxQx)
569 390
570 391 .int L(P0Q9)-L(fwdPxQx) /* 72 */
571 392 .int L(P1Q9)-L(fwdPxQx)
572 393 .int L(P2Q9)-L(fwdPxQx)
573 394 .int L(P3Q9)-L(fwdPxQx)
574 395 .int L(P4Q9)-L(fwdPxQx)
575 396 .int L(P5Q9)-L(fwdPxQx)
576 397 .int L(P6Q9)-L(fwdPxQx)
577 398 .int L(P7Q9)-L(fwdPxQx) /* 79 */
578 399
579 400 .p2align 4
580 401 L(P0Q9):
581 402 mov -0x48(%rdi), %rcx
582 403 mov %rcx, -0x48(%rsi)
583 404 L(P0Q8):
584 405 mov -0x40(%rdi), %r10
585 406 mov %r10, -0x40(%rsi)
586 407 L(P0Q7):
587 408 mov -0x38(%rdi), %r8
588 409 mov %r8, -0x38(%rsi)
589 410 L(P0Q6):
590 411 mov -0x30(%rdi), %rcx
591 412 mov %rcx, -0x30(%rsi)
592 413 L(P0Q5):
593 414 mov -0x28(%rdi), %r10
594 415 mov %r10, -0x28(%rsi)
595 416 L(P0Q4):
596 417 mov -0x20(%rdi), %r8
597 418 mov %r8, -0x20(%rsi)
598 419 L(P0Q3):
599 420 mov -0x18(%rdi), %rcx
600 421 mov %rcx, -0x18(%rsi)
601 422 L(P0Q2):
602 423 mov -0x10(%rdi), %r10
603 424 mov %r10, -0x10(%rsi)
604 425 L(P0Q1):
605 426 mov -0x8(%rdi), %r8
606 427 mov %r8, -0x8(%rsi)
607 428 L(P0Q0):
608 429 ret
609 430
610 431 .p2align 4
611 432 L(P1Q9):
612 433 mov -0x49(%rdi), %r8
613 434 mov %r8, -0x49(%rsi)
614 435 L(P1Q8):
615 436 mov -0x41(%rdi), %rcx
616 437 mov %rcx, -0x41(%rsi)
617 438 L(P1Q7):
618 439 mov -0x39(%rdi), %r10
619 440 mov %r10, -0x39(%rsi)
620 441 L(P1Q6):
621 442 mov -0x31(%rdi), %r8
622 443 mov %r8, -0x31(%rsi)
623 444 L(P1Q5):
624 445 mov -0x29(%rdi), %rcx
625 446 mov %rcx, -0x29(%rsi)
626 447 L(P1Q4):
627 448 mov -0x21(%rdi), %r10
628 449 mov %r10, -0x21(%rsi)
629 450 L(P1Q3):
630 451 mov -0x19(%rdi), %r8
631 452 mov %r8, -0x19(%rsi)
632 453 L(P1Q2):
633 454 mov -0x11(%rdi), %rcx
634 455 mov %rcx, -0x11(%rsi)
635 456 L(P1Q1):
636 457 mov -0x9(%rdi), %r10
637 458 mov %r10, -0x9(%rsi)
638 459 L(P1Q0):
639 460 movzbq -0x1(%rdi), %r8
640 461 mov %r8b, -0x1(%rsi)
641 462 ret
642 463
643 464 .p2align 4
644 465 L(P2Q9):
645 466 mov -0x4a(%rdi), %r8
646 467 mov %r8, -0x4a(%rsi)
647 468 L(P2Q8):
648 469 mov -0x42(%rdi), %rcx
649 470 mov %rcx, -0x42(%rsi)
650 471 L(P2Q7):
651 472 mov -0x3a(%rdi), %r10
652 473 mov %r10, -0x3a(%rsi)
653 474 L(P2Q6):
654 475 mov -0x32(%rdi), %r8
655 476 mov %r8, -0x32(%rsi)
656 477 L(P2Q5):
657 478 mov -0x2a(%rdi), %rcx
658 479 mov %rcx, -0x2a(%rsi)
659 480 L(P2Q4):
660 481 mov -0x22(%rdi), %r10
661 482 mov %r10, -0x22(%rsi)
662 483 L(P2Q3):
663 484 mov -0x1a(%rdi), %r8
664 485 mov %r8, -0x1a(%rsi)
665 486 L(P2Q2):
666 487 mov -0x12(%rdi), %rcx
667 488 mov %rcx, -0x12(%rsi)
668 489 L(P2Q1):
669 490 mov -0xa(%rdi), %r10
670 491 mov %r10, -0xa(%rsi)
671 492 L(P2Q0):
672 493 movzwq -0x2(%rdi), %r8
673 494 mov %r8w, -0x2(%rsi)
674 495 ret
675 496
676 497 .p2align 4
677 498 L(P3Q9):
678 499 mov -0x4b(%rdi), %r8
679 500 mov %r8, -0x4b(%rsi)
680 501 L(P3Q8):
681 502 mov -0x43(%rdi), %rcx
682 503 mov %rcx, -0x43(%rsi)
683 504 L(P3Q7):
684 505 mov -0x3b(%rdi), %r10
685 506 mov %r10, -0x3b(%rsi)
686 507 L(P3Q6):
687 508 mov -0x33(%rdi), %r8
688 509 mov %r8, -0x33(%rsi)
689 510 L(P3Q5):
690 511 mov -0x2b(%rdi), %rcx
691 512 mov %rcx, -0x2b(%rsi)
692 513 L(P3Q4):
693 514 mov -0x23(%rdi), %r10
694 515 mov %r10, -0x23(%rsi)
695 516 L(P3Q3):
696 517 mov -0x1b(%rdi), %r8
697 518 mov %r8, -0x1b(%rsi)
698 519 L(P3Q2):
699 520 mov -0x13(%rdi), %rcx
700 521 mov %rcx, -0x13(%rsi)
701 522 L(P3Q1):
702 523 mov -0xb(%rdi), %r10
703 524 mov %r10, -0xb(%rsi)
704 525 /*
705 526 * These trailing loads/stores have to do all their loads 1st,
706 527 * then do the stores.
707 528 */
708 529 L(P3Q0):
709 530 movzwq -0x3(%rdi), %r8
710 531 movzbq -0x1(%rdi), %r10
711 532 mov %r8w, -0x3(%rsi)
712 533 mov %r10b, -0x1(%rsi)
713 534 ret
714 535
715 536 .p2align 4
716 537 L(P4Q9):
717 538 mov -0x4c(%rdi), %r8
718 539 mov %r8, -0x4c(%rsi)
719 540 L(P4Q8):
720 541 mov -0x44(%rdi), %rcx
721 542 mov %rcx, -0x44(%rsi)
722 543 L(P4Q7):
723 544 mov -0x3c(%rdi), %r10
724 545 mov %r10, -0x3c(%rsi)
725 546 L(P4Q6):
726 547 mov -0x34(%rdi), %r8
727 548 mov %r8, -0x34(%rsi)
728 549 L(P4Q5):
729 550 mov -0x2c(%rdi), %rcx
730 551 mov %rcx, -0x2c(%rsi)
731 552 L(P4Q4):
732 553 mov -0x24(%rdi), %r10
733 554 mov %r10, -0x24(%rsi)
734 555 L(P4Q3):
735 556 mov -0x1c(%rdi), %r8
736 557 mov %r8, -0x1c(%rsi)
737 558 L(P4Q2):
738 559 mov -0x14(%rdi), %rcx
739 560 mov %rcx, -0x14(%rsi)
740 561 L(P4Q1):
741 562 mov -0xc(%rdi), %r10
742 563 mov %r10, -0xc(%rsi)
743 564 L(P4Q0):
744 565 mov -0x4(%rdi), %r8d
745 566 mov %r8d, -0x4(%rsi)
746 567 ret
747 568
748 569 .p2align 4
749 570 L(P5Q9):
750 571 mov -0x4d(%rdi), %r8
751 572 mov %r8, -0x4d(%rsi)
752 573 L(P5Q8):
753 574 mov -0x45(%rdi), %rcx
754 575 mov %rcx, -0x45(%rsi)
755 576 L(P5Q7):
756 577 mov -0x3d(%rdi), %r10
757 578 mov %r10, -0x3d(%rsi)
758 579 L(P5Q6):
759 580 mov -0x35(%rdi), %r8
760 581 mov %r8, -0x35(%rsi)
761 582 L(P5Q5):
762 583 mov -0x2d(%rdi), %rcx
763 584 mov %rcx, -0x2d(%rsi)
764 585 L(P5Q4):
765 586 mov -0x25(%rdi), %r10
766 587 mov %r10, -0x25(%rsi)
767 588 L(P5Q3):
768 589 mov -0x1d(%rdi), %r8
769 590 mov %r8, -0x1d(%rsi)
770 591 L(P5Q2):
771 592 mov -0x15(%rdi), %rcx
772 593 mov %rcx, -0x15(%rsi)
773 594 L(P5Q1):
774 595 mov -0xd(%rdi), %r10
775 596 mov %r10, -0xd(%rsi)
776 597 L(P5Q0):
777 598 mov -0x5(%rdi), %r8d
778 599 movzbq -0x1(%rdi), %r10
779 600 mov %r8d, -0x5(%rsi)
780 601 mov %r10b, -0x1(%rsi)
781 602 ret
782 603
783 604 .p2align 4
784 605 L(P6Q9):
785 606 mov -0x4e(%rdi), %r8
786 607 mov %r8, -0x4e(%rsi)
787 608 L(P6Q8):
788 609 mov -0x46(%rdi), %rcx
789 610 mov %rcx, -0x46(%rsi)
790 611 L(P6Q7):
791 612 mov -0x3e(%rdi), %r10
792 613 mov %r10, -0x3e(%rsi)
793 614 L(P6Q6):
794 615 mov -0x36(%rdi), %r8
795 616 mov %r8, -0x36(%rsi)
796 617 L(P6Q5):
797 618 mov -0x2e(%rdi), %rcx
798 619 mov %rcx, -0x2e(%rsi)
799 620 L(P6Q4):
800 621 mov -0x26(%rdi), %r10
801 622 mov %r10, -0x26(%rsi)
802 623 L(P6Q3):
803 624 mov -0x1e(%rdi), %r8
804 625 mov %r8, -0x1e(%rsi)
805 626 L(P6Q2):
806 627 mov -0x16(%rdi), %rcx
807 628 mov %rcx, -0x16(%rsi)
808 629 L(P6Q1):
809 630 mov -0xe(%rdi), %r10
810 631 mov %r10, -0xe(%rsi)
811 632 L(P6Q0):
812 633 mov -0x6(%rdi), %r8d
813 634 movzwq -0x2(%rdi), %r10
814 635 mov %r8d, -0x6(%rsi)
815 636 mov %r10w, -0x2(%rsi)
816 637 ret
817 638
818 639 .p2align 4
819 640 L(P7Q9):
820 641 mov -0x4f(%rdi), %r8
821 642 mov %r8, -0x4f(%rsi)
822 643 L(P7Q8):
823 644 mov -0x47(%rdi), %rcx
824 645 mov %rcx, -0x47(%rsi)
825 646 L(P7Q7):
826 647 mov -0x3f(%rdi), %r10
827 648 mov %r10, -0x3f(%rsi)
828 649 L(P7Q6):
829 650 mov -0x37(%rdi), %r8
830 651 mov %r8, -0x37(%rsi)
831 652 L(P7Q5):
832 653 mov -0x2f(%rdi), %rcx
833 654 mov %rcx, -0x2f(%rsi)
834 655 L(P7Q4):
835 656 mov -0x27(%rdi), %r10
836 657 mov %r10, -0x27(%rsi)
837 658 L(P7Q3):
838 659 mov -0x1f(%rdi), %r8
839 660 mov %r8, -0x1f(%rsi)
840 661 L(P7Q2):
841 662 mov -0x17(%rdi), %rcx
842 663 mov %rcx, -0x17(%rsi)
843 664 L(P7Q1):
844 665 mov -0xf(%rdi), %r10
845 666 mov %r10, -0xf(%rsi)
846 667 L(P7Q0):
847 668 mov -0x7(%rdi), %r8d
848 669 movzwq -0x3(%rdi), %r10
849 670 movzbq -0x1(%rdi), %rcx
850 671 mov %r8d, -0x7(%rsi)
851 672 mov %r10w, -0x3(%rsi)
852 673 mov %cl, -0x1(%rsi)
853 674 ret
854 675
855 676 /*
856 677 * For large sizes rep smovq is fastest.
857 678 * Transition point determined experimentally as measured on
858 679 * Intel Xeon processors (incl. Nehalem and previous generations) and
859 680 * AMD Opteron. The transition value is patched at boot time to avoid
860 681 * memory reference hit.
861 682 */
862 683 .globl bcopy_patch_start
863 684 bcopy_patch_start:
864 685 cmpq $BCOPY_NHM_REP, %rdx
865 686 .globl bcopy_patch_end
866 687 bcopy_patch_end:
867 688
868 689 .p2align 4
869 690 ALTENTRY(bcopy_ck_size)
870 691
871 692 cmpq $BCOPY_DFLT_REP, %rdx
872 693 jae L(use_rep)
873 694
874 695 /*
875 696 * Align to a 8-byte boundary. Avoids penalties from unaligned stores
876 697 * as well as from stores spanning cachelines.
877 698 */
878 699 test $0x7, %rsi
879 700 jz L(aligned_loop)
880 701 test $0x1, %rsi
881 702 jz 2f
882 703 movzbq (%rdi), %r8
883 704 dec %rdx
884 705 inc %rdi
885 706 mov %r8b, (%rsi)
886 707 inc %rsi
887 708 2:
888 709 test $0x2, %rsi
889 710 jz 4f
890 711 movzwq (%rdi), %r8
891 712 sub $0x2, %rdx
892 713 add $0x2, %rdi
893 714 mov %r8w, (%rsi)
894 715 add $0x2, %rsi
895 716 4:
896 717 test $0x4, %rsi
897 718 jz L(aligned_loop)
898 719 mov (%rdi), %r8d
899 720 sub $0x4, %rdx
900 721 add $0x4, %rdi
901 722 mov %r8d, (%rsi)
902 723 add $0x4, %rsi
903 724
904 725 /*
905 726 * Copy 64-bytes per loop
906 727 */
907 728 .p2align 4
908 729 L(aligned_loop):
909 730 mov (%rdi), %r8
910 731 mov 0x8(%rdi), %r10
911 732 lea -0x40(%rdx), %rdx
912 733 mov %r8, (%rsi)
913 734 mov %r10, 0x8(%rsi)
914 735 mov 0x10(%rdi), %rcx
915 736 mov 0x18(%rdi), %r8
916 737 mov %rcx, 0x10(%rsi)
917 738 mov %r8, 0x18(%rsi)
918 739
919 740 cmp $0x40, %rdx
920 741 mov 0x20(%rdi), %r10
921 742 mov 0x28(%rdi), %rcx
922 743 mov %r10, 0x20(%rsi)
923 744 mov %rcx, 0x28(%rsi)
924 745 mov 0x30(%rdi), %r8
925 746 mov 0x38(%rdi), %r10
926 747 lea 0x40(%rdi), %rdi
927 748 mov %r8, 0x30(%rsi)
928 749 mov %r10, 0x38(%rsi)
929 750 lea 0x40(%rsi), %rsi
930 751 jae L(aligned_loop)
931 752
932 753 /*
933 754 * Copy remaining bytes (0-63)
934 755 */
935 756 L(do_remainder):
936 757 leaq L(fwdPxQx)(%rip), %r10
937 758 addq %rdx, %rdi
938 759 addq %rdx, %rsi
939 760 movslq (%r10,%rdx,4), %rcx
940 761 leaq (%rcx,%r10,1), %r10
941 762 INDIRECT_JMP_REG(r10)
942 763
943 764 /*
944 765 * Use rep smovq. Clear remainder via unrolled code
945 766 */
946 767 .p2align 4
947 768 L(use_rep):
948 769 xchgq %rdi, %rsi /* %rsi = source, %rdi = destination */
949 770 movq %rdx, %rcx /* %rcx = count */
950 771 shrq $3, %rcx /* 8-byte word count */
951 772 rep
952 773 smovq
953 774
954 775 xchgq %rsi, %rdi /* %rdi = src, %rsi = destination */
955 776 andq $7, %rdx /* remainder */
956 777 jnz L(do_remainder)
957 778 ret
958 779 #undef L
959 780 SET_SIZE(bcopy_ck_size)
960 781
961 782 #ifdef DEBUG
962 783 /*
963 784 * Setup frame on the run-time stack. The end of the input argument
964 785 * area must be aligned on a 16 byte boundary. The stack pointer %rsp,
965 786 * always points to the end of the latest allocated stack frame.
966 787 * panic(const char *format, ...) is a varargs function. When a
967 788 * function taking variable arguments is called, %rax must be set
968 789 * to eight times the number of floating point parameters passed
969 790 * to the function in SSE registers.
|
↓ open down ↓ |
508 lines elided |
↑ open up ↑ |
970 791 */
971 792 call_panic:
972 793 pushq %rbp /* align stack properly */
973 794 movq %rsp, %rbp
974 795 xorl %eax, %eax /* no variable arguments */
975 796 call panic /* %rdi = format string */
976 797 #endif
977 798 SET_SIZE(bcopy_altentry)
978 799 SET_SIZE(bcopy)
979 800
980 -#elif defined(__i386)
981 801
982 -#define ARG_FROM 4
983 -#define ARG_TO 8
984 -#define ARG_COUNT 12
985 -
986 - ENTRY(bcopy)
987 -#ifdef DEBUG
988 - movl ARG_COUNT(%esp), %eax
989 - orl %eax, %eax
990 - jz 1f
991 - movl postbootkernelbase, %eax
992 - cmpl %eax, ARG_FROM(%esp)
993 - jb 0f
994 - cmpl %eax, ARG_TO(%esp)
995 - jnb 1f
996 -0: pushl %ebp
997 - movl %esp, %ebp
998 - pushl $.bcopy_panic_msg
999 - call panic
1000 -1:
1001 -#endif
1002 -do_copy:
1003 - movl %esi, %eax /* save registers */
1004 - movl %edi, %edx
1005 - movl ARG_COUNT(%esp), %ecx
1006 - movl ARG_FROM(%esp), %esi
1007 - movl ARG_TO(%esp), %edi
1008 -
1009 - shrl $2, %ecx /* word count */
1010 - rep
1011 - smovl
1012 - movl ARG_COUNT(%esp), %ecx
1013 - andl $3, %ecx /* bytes left over */
1014 - rep
1015 - smovb
1016 - movl %eax, %esi /* restore registers */
1017 - movl %edx, %edi
1018 - ret
1019 - SET_SIZE(bcopy)
1020 -
1021 -#undef ARG_COUNT
1022 -#undef ARG_FROM
1023 -#undef ARG_TO
1024 -
1025 -#endif /* __i386 */
1026 -#endif /* __lint */
1027 -
1028 -
1029 802 /*
1030 803 * Zero a block of storage, returning an error code if we
1031 804 * take a kernel pagefault which cannot be resolved.
1032 805 * Returns errno value on pagefault error, 0 if all ok
1033 806 */
1034 807
1035 -#if defined(__lint)
1036 -
1037 -/* ARGSUSED */
1038 -int
1039 -kzero(void *addr, size_t count)
1040 -{ return (0); }
1041 -
1042 -#else /* __lint */
1043 -
1044 -#if defined(__amd64)
1045 -
1046 808 ENTRY(kzero)
1047 809 #ifdef DEBUG
1048 810 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */
1049 811 jnb 0f
1050 812 leaq .kzero_panic_msg(%rip), %rdi
1051 813 jmp call_panic /* setup stack and call panic */
1052 814 0:
1053 815 #endif
1054 816 /*
1055 817 * pass lofault value as 3rd argument for fault return
1056 818 */
1057 819 leaq _kzeroerr(%rip), %rdx
1058 820
1059 821 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
1060 822 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
1061 823 movq %rdx, T_LOFAULT(%r9) /* new lofault */
1062 824 call bzero_altentry
1063 825 xorl %eax, %eax
1064 826 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
1065 827 ret
|
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
1066 828 /*
1067 829 * A fault during bzero is indicated through an errno value
1068 830 * in %rax when we iretq to here.
1069 831 */
1070 832 _kzeroerr:
1071 833 addq $8, %rsp /* pop bzero_altentry call ret addr */
1072 834 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
1073 835 ret
1074 836 SET_SIZE(kzero)
1075 837
1076 -#elif defined(__i386)
1077 -
1078 -#define ARG_ADDR 8
1079 -#define ARG_COUNT 12
1080 -
1081 - ENTRY(kzero)
1082 -#ifdef DEBUG
1083 - pushl %ebp
1084 - movl %esp, %ebp
1085 - movl postbootkernelbase, %eax
1086 - cmpl %eax, ARG_ADDR(%ebp)
1087 - jnb 0f
1088 - pushl $.kzero_panic_msg
1089 - call panic
1090 -0: popl %ebp
1091 -#endif
1092 - lea _kzeroerr, %eax /* kzeroerr is lofault value */
1093 -
1094 - pushl %ebp /* save stack base */
1095 - movl %esp, %ebp /* set new stack base */
1096 - pushl %edi /* save %edi */
1097 -
1098 - mov %gs:CPU_THREAD, %edx
1099 - movl T_LOFAULT(%edx), %edi
1100 - pushl %edi /* save the current lofault */
1101 - movl %eax, T_LOFAULT(%edx) /* new lofault */
1102 -
1103 - movl ARG_COUNT(%ebp), %ecx /* get size in bytes */
1104 - movl ARG_ADDR(%ebp), %edi /* %edi <- address of bytes to clear */
1105 - shrl $2, %ecx /* Count of double words to zero */
1106 - xorl %eax, %eax /* sstol val */
1107 - rep
1108 - sstol /* %ecx contains words to clear (%eax=0) */
1109 -
1110 - movl ARG_COUNT(%ebp), %ecx /* get size in bytes */
1111 - andl $3, %ecx /* do mod 4 */
1112 - rep
1113 - sstob /* %ecx contains residual bytes to clear */
1114 -
1115 - /*
1116 - * A fault during kzero is indicated through an errno value
1117 - * in %eax when we iret to here.
1118 - */
1119 -_kzeroerr:
1120 - popl %edi
1121 - movl %edi, T_LOFAULT(%edx) /* restore the original lofault */
1122 - popl %edi
1123 - popl %ebp
1124 - ret
1125 - SET_SIZE(kzero)
1126 -
1127 -#undef ARG_ADDR
1128 -#undef ARG_COUNT
1129 -
1130 -#endif /* __i386 */
1131 -#endif /* __lint */
1132 -
1133 838 /*
1134 839 * Zero a block of storage.
1135 840 */
1136 841
1137 -#if defined(__lint)
1138 -
1139 -/* ARGSUSED */
1140 -void
1141 -bzero(void *addr, size_t count)
1142 -{}
1143 -
1144 -#else /* __lint */
1145 -
1146 -#if defined(__amd64)
1147 -
1148 842 ENTRY(bzero)
1149 843 #ifdef DEBUG
1150 844 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */
1151 845 jnb 0f
1152 846 leaq .bzero_panic_msg(%rip), %rdi
1153 847 jmp call_panic /* setup stack and call panic */
1154 848 0:
1155 849 #endif
1156 850 ALTENTRY(bzero_altentry)
1157 851 do_zero:
1158 852 #define L(s) .bzero/**/s
1159 853 xorl %eax, %eax
1160 854
1161 855 cmpq $0x50, %rsi /* 80 */
1162 856 jae L(ck_align)
1163 857
1164 858 /*
1165 859 * Performance data shows many caller's are zeroing small buffers. So
1166 860 * for best perf for these sizes unrolled code is used. Store zeros
1167 861 * without worrying about alignment.
1168 862 */
1169 863 leaq L(setPxQx)(%rip), %r10
1170 864 addq %rsi, %rdi
1171 865 movslq (%r10,%rsi,4), %rcx
1172 866 leaq (%rcx,%r10,1), %r10
1173 867 INDIRECT_JMP_REG(r10)
1174 868
1175 869 .p2align 4
1176 870 L(setPxQx):
1177 871 .int L(P0Q0)-L(setPxQx) /* 0 */
1178 872 .int L(P1Q0)-L(setPxQx)
1179 873 .int L(P2Q0)-L(setPxQx)
1180 874 .int L(P3Q0)-L(setPxQx)
1181 875 .int L(P4Q0)-L(setPxQx)
1182 876 .int L(P5Q0)-L(setPxQx)
1183 877 .int L(P6Q0)-L(setPxQx)
1184 878 .int L(P7Q0)-L(setPxQx)
1185 879
1186 880 .int L(P0Q1)-L(setPxQx) /* 8 */
1187 881 .int L(P1Q1)-L(setPxQx)
1188 882 .int L(P2Q1)-L(setPxQx)
1189 883 .int L(P3Q1)-L(setPxQx)
1190 884 .int L(P4Q1)-L(setPxQx)
1191 885 .int L(P5Q1)-L(setPxQx)
1192 886 .int L(P6Q1)-L(setPxQx)
1193 887 .int L(P7Q1)-L(setPxQx)
1194 888
1195 889 .int L(P0Q2)-L(setPxQx) /* 16 */
1196 890 .int L(P1Q2)-L(setPxQx)
1197 891 .int L(P2Q2)-L(setPxQx)
1198 892 .int L(P3Q2)-L(setPxQx)
1199 893 .int L(P4Q2)-L(setPxQx)
1200 894 .int L(P5Q2)-L(setPxQx)
1201 895 .int L(P6Q2)-L(setPxQx)
1202 896 .int L(P7Q2)-L(setPxQx)
1203 897
1204 898 .int L(P0Q3)-L(setPxQx) /* 24 */
1205 899 .int L(P1Q3)-L(setPxQx)
1206 900 .int L(P2Q3)-L(setPxQx)
1207 901 .int L(P3Q3)-L(setPxQx)
1208 902 .int L(P4Q3)-L(setPxQx)
1209 903 .int L(P5Q3)-L(setPxQx)
1210 904 .int L(P6Q3)-L(setPxQx)
1211 905 .int L(P7Q3)-L(setPxQx)
1212 906
1213 907 .int L(P0Q4)-L(setPxQx) /* 32 */
1214 908 .int L(P1Q4)-L(setPxQx)
1215 909 .int L(P2Q4)-L(setPxQx)
1216 910 .int L(P3Q4)-L(setPxQx)
1217 911 .int L(P4Q4)-L(setPxQx)
1218 912 .int L(P5Q4)-L(setPxQx)
1219 913 .int L(P6Q4)-L(setPxQx)
1220 914 .int L(P7Q4)-L(setPxQx)
1221 915
1222 916 .int L(P0Q5)-L(setPxQx) /* 40 */
1223 917 .int L(P1Q5)-L(setPxQx)
1224 918 .int L(P2Q5)-L(setPxQx)
1225 919 .int L(P3Q5)-L(setPxQx)
1226 920 .int L(P4Q5)-L(setPxQx)
1227 921 .int L(P5Q5)-L(setPxQx)
1228 922 .int L(P6Q5)-L(setPxQx)
1229 923 .int L(P7Q5)-L(setPxQx)
1230 924
1231 925 .int L(P0Q6)-L(setPxQx) /* 48 */
1232 926 .int L(P1Q6)-L(setPxQx)
1233 927 .int L(P2Q6)-L(setPxQx)
1234 928 .int L(P3Q6)-L(setPxQx)
1235 929 .int L(P4Q6)-L(setPxQx)
1236 930 .int L(P5Q6)-L(setPxQx)
1237 931 .int L(P6Q6)-L(setPxQx)
1238 932 .int L(P7Q6)-L(setPxQx)
1239 933
1240 934 .int L(P0Q7)-L(setPxQx) /* 56 */
1241 935 .int L(P1Q7)-L(setPxQx)
1242 936 .int L(P2Q7)-L(setPxQx)
1243 937 .int L(P3Q7)-L(setPxQx)
1244 938 .int L(P4Q7)-L(setPxQx)
1245 939 .int L(P5Q7)-L(setPxQx)
1246 940 .int L(P6Q7)-L(setPxQx)
1247 941 .int L(P7Q7)-L(setPxQx)
1248 942
1249 943 .int L(P0Q8)-L(setPxQx) /* 64 */
1250 944 .int L(P1Q8)-L(setPxQx)
1251 945 .int L(P2Q8)-L(setPxQx)
1252 946 .int L(P3Q8)-L(setPxQx)
1253 947 .int L(P4Q8)-L(setPxQx)
1254 948 .int L(P5Q8)-L(setPxQx)
1255 949 .int L(P6Q8)-L(setPxQx)
1256 950 .int L(P7Q8)-L(setPxQx)
1257 951
1258 952 .int L(P0Q9)-L(setPxQx) /* 72 */
1259 953 .int L(P1Q9)-L(setPxQx)
1260 954 .int L(P2Q9)-L(setPxQx)
1261 955 .int L(P3Q9)-L(setPxQx)
1262 956 .int L(P4Q9)-L(setPxQx)
1263 957 .int L(P5Q9)-L(setPxQx)
1264 958 .int L(P6Q9)-L(setPxQx)
1265 959 .int L(P7Q9)-L(setPxQx) /* 79 */
1266 960
1267 961 .p2align 4
1268 962 L(P0Q9): mov %rax, -0x48(%rdi)
1269 963 L(P0Q8): mov %rax, -0x40(%rdi)
1270 964 L(P0Q7): mov %rax, -0x38(%rdi)
1271 965 L(P0Q6): mov %rax, -0x30(%rdi)
1272 966 L(P0Q5): mov %rax, -0x28(%rdi)
1273 967 L(P0Q4): mov %rax, -0x20(%rdi)
1274 968 L(P0Q3): mov %rax, -0x18(%rdi)
1275 969 L(P0Q2): mov %rax, -0x10(%rdi)
1276 970 L(P0Q1): mov %rax, -0x8(%rdi)
1277 971 L(P0Q0):
1278 972 ret
1279 973
1280 974 .p2align 4
1281 975 L(P1Q9): mov %rax, -0x49(%rdi)
1282 976 L(P1Q8): mov %rax, -0x41(%rdi)
1283 977 L(P1Q7): mov %rax, -0x39(%rdi)
1284 978 L(P1Q6): mov %rax, -0x31(%rdi)
1285 979 L(P1Q5): mov %rax, -0x29(%rdi)
1286 980 L(P1Q4): mov %rax, -0x21(%rdi)
1287 981 L(P1Q3): mov %rax, -0x19(%rdi)
1288 982 L(P1Q2): mov %rax, -0x11(%rdi)
1289 983 L(P1Q1): mov %rax, -0x9(%rdi)
1290 984 L(P1Q0): mov %al, -0x1(%rdi)
1291 985 ret
1292 986
1293 987 .p2align 4
1294 988 L(P2Q9): mov %rax, -0x4a(%rdi)
1295 989 L(P2Q8): mov %rax, -0x42(%rdi)
1296 990 L(P2Q7): mov %rax, -0x3a(%rdi)
1297 991 L(P2Q6): mov %rax, -0x32(%rdi)
1298 992 L(P2Q5): mov %rax, -0x2a(%rdi)
1299 993 L(P2Q4): mov %rax, -0x22(%rdi)
1300 994 L(P2Q3): mov %rax, -0x1a(%rdi)
1301 995 L(P2Q2): mov %rax, -0x12(%rdi)
1302 996 L(P2Q1): mov %rax, -0xa(%rdi)
1303 997 L(P2Q0): mov %ax, -0x2(%rdi)
1304 998 ret
1305 999
1306 1000 .p2align 4
1307 1001 L(P3Q9): mov %rax, -0x4b(%rdi)
1308 1002 L(P3Q8): mov %rax, -0x43(%rdi)
1309 1003 L(P3Q7): mov %rax, -0x3b(%rdi)
1310 1004 L(P3Q6): mov %rax, -0x33(%rdi)
1311 1005 L(P3Q5): mov %rax, -0x2b(%rdi)
1312 1006 L(P3Q4): mov %rax, -0x23(%rdi)
1313 1007 L(P3Q3): mov %rax, -0x1b(%rdi)
1314 1008 L(P3Q2): mov %rax, -0x13(%rdi)
1315 1009 L(P3Q1): mov %rax, -0xb(%rdi)
1316 1010 L(P3Q0): mov %ax, -0x3(%rdi)
1317 1011 mov %al, -0x1(%rdi)
1318 1012 ret
1319 1013
1320 1014 .p2align 4
1321 1015 L(P4Q9): mov %rax, -0x4c(%rdi)
1322 1016 L(P4Q8): mov %rax, -0x44(%rdi)
1323 1017 L(P4Q7): mov %rax, -0x3c(%rdi)
1324 1018 L(P4Q6): mov %rax, -0x34(%rdi)
1325 1019 L(P4Q5): mov %rax, -0x2c(%rdi)
1326 1020 L(P4Q4): mov %rax, -0x24(%rdi)
1327 1021 L(P4Q3): mov %rax, -0x1c(%rdi)
1328 1022 L(P4Q2): mov %rax, -0x14(%rdi)
1329 1023 L(P4Q1): mov %rax, -0xc(%rdi)
1330 1024 L(P4Q0): mov %eax, -0x4(%rdi)
1331 1025 ret
1332 1026
1333 1027 .p2align 4
1334 1028 L(P5Q9): mov %rax, -0x4d(%rdi)
1335 1029 L(P5Q8): mov %rax, -0x45(%rdi)
1336 1030 L(P5Q7): mov %rax, -0x3d(%rdi)
1337 1031 L(P5Q6): mov %rax, -0x35(%rdi)
1338 1032 L(P5Q5): mov %rax, -0x2d(%rdi)
1339 1033 L(P5Q4): mov %rax, -0x25(%rdi)
1340 1034 L(P5Q3): mov %rax, -0x1d(%rdi)
1341 1035 L(P5Q2): mov %rax, -0x15(%rdi)
1342 1036 L(P5Q1): mov %rax, -0xd(%rdi)
1343 1037 L(P5Q0): mov %eax, -0x5(%rdi)
1344 1038 mov %al, -0x1(%rdi)
1345 1039 ret
1346 1040
1347 1041 .p2align 4
1348 1042 L(P6Q9): mov %rax, -0x4e(%rdi)
1349 1043 L(P6Q8): mov %rax, -0x46(%rdi)
1350 1044 L(P6Q7): mov %rax, -0x3e(%rdi)
1351 1045 L(P6Q6): mov %rax, -0x36(%rdi)
1352 1046 L(P6Q5): mov %rax, -0x2e(%rdi)
1353 1047 L(P6Q4): mov %rax, -0x26(%rdi)
1354 1048 L(P6Q3): mov %rax, -0x1e(%rdi)
1355 1049 L(P6Q2): mov %rax, -0x16(%rdi)
1356 1050 L(P6Q1): mov %rax, -0xe(%rdi)
1357 1051 L(P6Q0): mov %eax, -0x6(%rdi)
1358 1052 mov %ax, -0x2(%rdi)
1359 1053 ret
1360 1054
1361 1055 .p2align 4
1362 1056 L(P7Q9): mov %rax, -0x4f(%rdi)
1363 1057 L(P7Q8): mov %rax, -0x47(%rdi)
1364 1058 L(P7Q7): mov %rax, -0x3f(%rdi)
1365 1059 L(P7Q6): mov %rax, -0x37(%rdi)
1366 1060 L(P7Q5): mov %rax, -0x2f(%rdi)
1367 1061 L(P7Q4): mov %rax, -0x27(%rdi)
1368 1062 L(P7Q3): mov %rax, -0x1f(%rdi)
1369 1063 L(P7Q2): mov %rax, -0x17(%rdi)
1370 1064 L(P7Q1): mov %rax, -0xf(%rdi)
1371 1065 L(P7Q0): mov %eax, -0x7(%rdi)
1372 1066 mov %ax, -0x3(%rdi)
1373 1067 mov %al, -0x1(%rdi)
1374 1068 ret
1375 1069
1376 1070 /*
1377 1071 * Align to a 16-byte boundary. Avoids penalties from unaligned stores
1378 1072 * as well as from stores spanning cachelines. Note 16-byte alignment
1379 1073 * is better in case where rep sstosq is used.
1380 1074 */
1381 1075 .p2align 4
1382 1076 L(ck_align):
1383 1077 test $0xf, %rdi
1384 1078 jz L(aligned_now)
1385 1079 test $1, %rdi
1386 1080 jz 2f
1387 1081 mov %al, (%rdi)
1388 1082 dec %rsi
1389 1083 lea 1(%rdi),%rdi
1390 1084 2:
1391 1085 test $2, %rdi
1392 1086 jz 4f
1393 1087 mov %ax, (%rdi)
1394 1088 sub $2, %rsi
1395 1089 lea 2(%rdi),%rdi
1396 1090 4:
1397 1091 test $4, %rdi
1398 1092 jz 8f
1399 1093 mov %eax, (%rdi)
1400 1094 sub $4, %rsi
1401 1095 lea 4(%rdi),%rdi
1402 1096 8:
1403 1097 test $8, %rdi
1404 1098 jz L(aligned_now)
1405 1099 mov %rax, (%rdi)
1406 1100 sub $8, %rsi
1407 1101 lea 8(%rdi),%rdi
1408 1102
1409 1103 /*
1410 1104 * For large sizes rep sstoq is fastest.
1411 1105 * Transition point determined experimentally as measured on
1412 1106 * Intel Xeon processors (incl. Nehalem) and AMD Opteron.
1413 1107 */
1414 1108 L(aligned_now):
1415 1109 cmp $BZERO_USE_REP, %rsi
1416 1110 ja L(use_rep)
1417 1111
1418 1112 /*
1419 1113 * zero 64-bytes per loop
1420 1114 */
1421 1115 .p2align 4
1422 1116 L(bzero_loop):
1423 1117 leaq -0x40(%rsi), %rsi
1424 1118 cmpq $0x40, %rsi
1425 1119 movq %rax, (%rdi)
1426 1120 movq %rax, 0x8(%rdi)
1427 1121 movq %rax, 0x10(%rdi)
1428 1122 movq %rax, 0x18(%rdi)
1429 1123 movq %rax, 0x20(%rdi)
1430 1124 movq %rax, 0x28(%rdi)
1431 1125 movq %rax, 0x30(%rdi)
1432 1126 movq %rax, 0x38(%rdi)
1433 1127 leaq 0x40(%rdi), %rdi
1434 1128 jae L(bzero_loop)
1435 1129
1436 1130 /*
1437 1131 * Clear any remaining bytes..
1438 1132 */
1439 1133 9:
1440 1134 leaq L(setPxQx)(%rip), %r10
1441 1135 addq %rsi, %rdi
1442 1136 movslq (%r10,%rsi,4), %rcx
1443 1137 leaq (%rcx,%r10,1), %r10
1444 1138 INDIRECT_JMP_REG(r10)
1445 1139
1446 1140 /*
1447 1141 * Use rep sstoq. Clear any remainder via unrolled code
1448 1142 */
1449 1143 .p2align 4
1450 1144 L(use_rep):
1451 1145 movq %rsi, %rcx /* get size in bytes */
|
↓ open down ↓ |
294 lines elided |
↑ open up ↑ |
1452 1146 shrq $3, %rcx /* count of 8-byte words to zero */
1453 1147 rep
1454 1148 sstoq /* %rcx = words to clear (%rax=0) */
1455 1149 andq $7, %rsi /* remaining bytes */
1456 1150 jnz 9b
1457 1151 ret
1458 1152 #undef L
1459 1153 SET_SIZE(bzero_altentry)
1460 1154 SET_SIZE(bzero)
1461 1155
1462 -#elif defined(__i386)
1463 -
1464 -#define ARG_ADDR 4
1465 -#define ARG_COUNT 8
1466 -
1467 - ENTRY(bzero)
1468 -#ifdef DEBUG
1469 - movl postbootkernelbase, %eax
1470 - cmpl %eax, ARG_ADDR(%esp)
1471 - jnb 0f
1472 - pushl %ebp
1473 - movl %esp, %ebp
1474 - pushl $.bzero_panic_msg
1475 - call panic
1476 -0:
1477 -#endif
1478 -do_zero:
1479 - movl %edi, %edx
1480 - movl ARG_COUNT(%esp), %ecx
1481 - movl ARG_ADDR(%esp), %edi
1482 - shrl $2, %ecx
1483 - xorl %eax, %eax
1484 - rep
1485 - sstol
1486 - movl ARG_COUNT(%esp), %ecx
1487 - andl $3, %ecx
1488 - rep
1489 - sstob
1490 - movl %edx, %edi
1491 - ret
1492 - SET_SIZE(bzero)
1493 -
1494 -#undef ARG_ADDR
1495 -#undef ARG_COUNT
1496 -
1497 -#endif /* __i386 */
1498 -#endif /* __lint */
1499 -
1500 1156 /*
1501 1157 * Transfer data to and from user space -
1502 1158 * Note that these routines can cause faults
1503 1159 * It is assumed that the kernel has nothing at
1504 1160 * less than KERNELBASE in the virtual address space.
1505 1161 *
1506 1162 * Note that copyin(9F) and copyout(9F) are part of the
1507 1163 * DDI/DKI which specifies that they return '-1' on "errors."
1508 1164 *
1509 1165 * Sigh.
1510 1166 *
1511 1167 * So there's two extremely similar routines - xcopyin_nta() and
|
↓ open down ↓ |
2 lines elided |
↑ open up ↑ |
1512 1168 * xcopyout_nta() which return the errno that we've faithfully computed.
1513 1169 * This allows other callers (e.g. uiomove(9F)) to work correctly.
1514 1170 * Given that these are used pretty heavily, we expand the calling
1515 1171 * sequences inline for all flavours (rather than making wrappers).
1516 1172 */
1517 1173
1518 1174 /*
1519 1175 * Copy user data to kernel space.
1520 1176 */
1521 1177
1522 -#if defined(__lint)
1523 -
1524 -/* ARGSUSED */
1525 -int
1526 -copyin(const void *uaddr, void *kaddr, size_t count)
1527 -{ return (0); }
1528 -
1529 -#else /* lint */
1530 -
1531 -#if defined(__amd64)
1532 -
1533 1178 ENTRY(copyin)
1534 1179 pushq %rbp
1535 1180 movq %rsp, %rbp
1536 1181 subq $24, %rsp
1537 1182
1538 1183 /*
1539 1184 * save args in case we trap and need to rerun as a copyop
1540 1185 */
1541 1186 movq %rdi, (%rsp)
1542 1187 movq %rsi, 0x8(%rsp)
1543 1188 movq %rdx, 0x10(%rsp)
1544 1189
1545 1190 movq kernelbase(%rip), %rax
1546 1191 #ifdef DEBUG
1547 1192 cmpq %rax, %rsi /* %rsi = kaddr */
1548 1193 jnb 1f
1549 1194 leaq .copyin_panic_msg(%rip), %rdi
1550 1195 xorl %eax, %eax
1551 1196 call panic
1552 1197 1:
1553 1198 #endif
1554 1199 /*
1555 1200 * pass lofault value as 4th argument to do_copy_fault
1556 1201 */
1557 1202 leaq _copyin_err(%rip), %rcx
1558 1203
1559 1204 movq %gs:CPU_THREAD, %r9
1560 1205 cmpq %rax, %rdi /* test uaddr < kernelbase */
1561 1206 jae 3f /* take copyop if uaddr > kernelbase */
1562 1207 SMAP_DISABLE_INSTR(0)
1563 1208 jmp do_copy_fault /* Takes care of leave for us */
1564 1209
1565 1210 _copyin_err:
1566 1211 SMAP_ENABLE_INSTR(2)
1567 1212 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
1568 1213 addq $8, %rsp /* pop bcopy_altentry call ret addr */
1569 1214 3:
1570 1215 movq T_COPYOPS(%r9), %rax
1571 1216 cmpq $0, %rax
1572 1217 jz 2f
1573 1218 /*
1574 1219 * reload args for the copyop
1575 1220 */
1576 1221 movq (%rsp), %rdi
1577 1222 movq 0x8(%rsp), %rsi
|
↓ open down ↓ |
35 lines elided |
↑ open up ↑ |
1578 1223 movq 0x10(%rsp), %rdx
1579 1224 leave
1580 1225 movq CP_COPYIN(%rax), %rax
1581 1226 INDIRECT_JMP_REG(rax)
1582 1227
1583 1228 2: movl $-1, %eax
1584 1229 leave
1585 1230 ret
1586 1231 SET_SIZE(copyin)
1587 1232
1588 -#elif defined(__i386)
1589 -
1590 -#define ARG_UADDR 4
1591 -#define ARG_KADDR 8
1592 -
1593 - ENTRY(copyin)
1594 - movl kernelbase, %ecx
1595 -#ifdef DEBUG
1596 - cmpl %ecx, ARG_KADDR(%esp)
1597 - jnb 1f
1598 - pushl %ebp
1599 - movl %esp, %ebp
1600 - pushl $.copyin_panic_msg
1601 - call panic
1602 -1:
1603 -#endif
1604 - lea _copyin_err, %eax
1605 -
1606 - movl %gs:CPU_THREAD, %edx
1607 - cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
1608 - jb do_copy_fault
1609 - jmp 3f
1610 -
1611 -_copyin_err:
1612 - popl %ecx
1613 - popl %edi
1614 - movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
1615 - popl %esi
1616 - popl %ebp
1617 -3:
1618 - movl T_COPYOPS(%edx), %eax
1619 - cmpl $0, %eax
1620 - jz 2f
1621 - jmp *CP_COPYIN(%eax)
1622 -
1623 -2: movl $-1, %eax
1624 - ret
1625 - SET_SIZE(copyin)
1626 -
1627 -#undef ARG_UADDR
1628 -#undef ARG_KADDR
1629 -
1630 -#endif /* __i386 */
1631 -#endif /* __lint */
1632 -
1633 -#if defined(__lint)
1634 -
1635 -/* ARGSUSED */
1636 -int
1637 -xcopyin_nta(const void *uaddr, void *kaddr, size_t count, int copy_cached)
1638 -{ return (0); }
1639 -
1640 -#else /* __lint */
1641 -
1642 -#if defined(__amd64)
1643 -
1644 1233 ENTRY(xcopyin_nta)
1645 1234 pushq %rbp
1646 1235 movq %rsp, %rbp
1647 1236 subq $24, %rsp
1648 1237
1649 1238 /*
1650 1239 * save args in case we trap and need to rerun as a copyop
1651 1240 * %rcx is consumed in this routine so we don't need to save
1652 1241 * it.
1653 1242 */
1654 1243 movq %rdi, (%rsp)
1655 1244 movq %rsi, 0x8(%rsp)
1656 1245 movq %rdx, 0x10(%rsp)
1657 1246
1658 1247 movq kernelbase(%rip), %rax
1659 1248 #ifdef DEBUG
1660 1249 cmpq %rax, %rsi /* %rsi = kaddr */
1661 1250 jnb 1f
1662 1251 leaq .xcopyin_panic_msg(%rip), %rdi
1663 1252 xorl %eax, %eax
1664 1253 call panic
1665 1254 1:
1666 1255 #endif
1667 1256 movq %gs:CPU_THREAD, %r9
1668 1257 cmpq %rax, %rdi /* test uaddr < kernelbase */
1669 1258 jae 4f
1670 1259 cmpq $0, %rcx /* No non-temporal access? */
1671 1260 /*
1672 1261 * pass lofault value as 4th argument to do_copy_fault
1673 1262 */
1674 1263 leaq _xcopyin_err(%rip), %rcx /* doesn't set rflags */
1675 1264 jnz 6f /* use regular access */
1676 1265 /*
1677 1266 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
1678 1267 */
1679 1268 cmpq $XCOPY_MIN_SIZE, %rdx
1680 1269 jae 5f
1681 1270 6:
1682 1271 SMAP_DISABLE_INSTR(1)
1683 1272 jmp do_copy_fault
1684 1273
1685 1274 /*
1686 1275 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
1687 1276 * count is COUNT_ALIGN_SIZE aligned.
1688 1277 */
1689 1278 5:
1690 1279 movq %rdi, %r10
1691 1280 orq %rsi, %r10
1692 1281 andq $NTA_ALIGN_MASK, %r10
1693 1282 orq %rdx, %r10
1694 1283 andq $COUNT_ALIGN_MASK, %r10
1695 1284 jnz 6b
1696 1285 leaq _xcopyin_nta_err(%rip), %rcx /* doesn't set rflags */
1697 1286 SMAP_DISABLE_INSTR(2)
1698 1287 jmp do_copy_fault_nta /* use non-temporal access */
1699 1288
1700 1289 4:
1701 1290 movl $EFAULT, %eax
1702 1291 jmp 3f
1703 1292
1704 1293 /*
1705 1294 * A fault during do_copy_fault or do_copy_fault_nta is
1706 1295 * indicated through an errno value in %rax and we iret from the
1707 1296 * trap handler to here.
1708 1297 */
1709 1298 _xcopyin_err:
1710 1299 addq $8, %rsp /* pop bcopy_altentry call ret addr */
1711 1300 _xcopyin_nta_err:
1712 1301 SMAP_ENABLE_INSTR(3)
1713 1302 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
1714 1303 3:
1715 1304 movq T_COPYOPS(%r9), %r8
1716 1305 cmpq $0, %r8
1717 1306 jz 2f
1718 1307
1719 1308 /*
1720 1309 * reload args for the copyop
1721 1310 */
1722 1311 movq (%rsp), %rdi
|
↓ open down ↓ |
69 lines elided |
↑ open up ↑ |
1723 1312 movq 0x8(%rsp), %rsi
1724 1313 movq 0x10(%rsp), %rdx
1725 1314 leave
1726 1315 movq CP_XCOPYIN(%r8), %r8
1727 1316 INDIRECT_JMP_REG(r8)
1728 1317
1729 1318 2: leave
1730 1319 ret
1731 1320 SET_SIZE(xcopyin_nta)
1732 1321
1733 -#elif defined(__i386)
1734 -
1735 -#define ARG_UADDR 4
1736 -#define ARG_KADDR 8
1737 -#define ARG_COUNT 12
1738 -#define ARG_CACHED 16
1739 -
1740 - .globl use_sse_copy
1741 -
1742 - ENTRY(xcopyin_nta)
1743 - movl kernelbase, %ecx
1744 - lea _xcopyin_err, %eax
1745 - movl %gs:CPU_THREAD, %edx
1746 - cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
1747 - jae 4f
1748 -
1749 - cmpl $0, use_sse_copy /* no sse support */
1750 - jz do_copy_fault
1751 -
1752 - cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */
1753 - jnz do_copy_fault
1754 -
1755 - /*
1756 - * Make sure cnt is >= XCOPY_MIN_SIZE bytes
1757 - */
1758 - cmpl $XCOPY_MIN_SIZE, ARG_COUNT(%esp)
1759 - jb do_copy_fault
1760 -
1761 - /*
1762 - * Make sure src and dst are NTA_ALIGN_SIZE aligned,
1763 - * count is COUNT_ALIGN_SIZE aligned.
1764 - */
1765 - movl ARG_UADDR(%esp), %ecx
1766 - orl ARG_KADDR(%esp), %ecx
1767 - andl $NTA_ALIGN_MASK, %ecx
1768 - orl ARG_COUNT(%esp), %ecx
1769 - andl $COUNT_ALIGN_MASK, %ecx
1770 - jnz do_copy_fault
1771 -
1772 - jmp do_copy_fault_nta /* use regular access */
1773 -
1774 -4:
1775 - movl $EFAULT, %eax
1776 - jmp 3f
1777 -
1778 - /*
1779 - * A fault during do_copy_fault or do_copy_fault_nta is
1780 - * indicated through an errno value in %eax and we iret from the
1781 - * trap handler to here.
1782 - */
1783 -_xcopyin_err:
1784 - popl %ecx
1785 - popl %edi
1786 - movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
1787 - popl %esi
1788 - popl %ebp
1789 -3:
1790 - cmpl $0, T_COPYOPS(%edx)
1791 - jz 2f
1792 - movl T_COPYOPS(%edx), %eax
1793 - jmp *CP_XCOPYIN(%eax)
1794 -
1795 -2: rep; ret /* use 2 byte return instruction when branch target */
1796 - /* AMD Software Optimization Guide - Section 6.2 */
1797 - SET_SIZE(xcopyin_nta)
1798 -
1799 -#undef ARG_UADDR
1800 -#undef ARG_KADDR
1801 -#undef ARG_COUNT
1802 -#undef ARG_CACHED
1803 -
1804 -#endif /* __i386 */
1805 -#endif /* __lint */
1806 -
1807 1322 /*
1808 1323 * Copy kernel data to user space.
1809 1324 */
1810 1325
1811 -#if defined(__lint)
1812 -
1813 -/* ARGSUSED */
1814 -int
1815 -copyout(const void *kaddr, void *uaddr, size_t count)
1816 -{ return (0); }
1817 -
1818 -#else /* __lint */
1819 -
1820 -#if defined(__amd64)
1821 -
1822 1326 ENTRY(copyout)
1823 1327 pushq %rbp
1824 1328 movq %rsp, %rbp
1825 1329 subq $24, %rsp
1826 1330
1827 1331 /*
1828 1332 * save args in case we trap and need to rerun as a copyop
1829 1333 */
1830 1334 movq %rdi, (%rsp)
1831 1335 movq %rsi, 0x8(%rsp)
1832 1336 movq %rdx, 0x10(%rsp)
1833 1337
1834 1338 movq kernelbase(%rip), %rax
1835 1339 #ifdef DEBUG
1836 1340 cmpq %rax, %rdi /* %rdi = kaddr */
1837 1341 jnb 1f
1838 1342 leaq .copyout_panic_msg(%rip), %rdi
1839 1343 xorl %eax, %eax
1840 1344 call panic
1841 1345 1:
1842 1346 #endif
1843 1347 /*
1844 1348 * pass lofault value as 4th argument to do_copy_fault
1845 1349 */
1846 1350 leaq _copyout_err(%rip), %rcx
1847 1351
1848 1352 movq %gs:CPU_THREAD, %r9
1849 1353 cmpq %rax, %rsi /* test uaddr < kernelbase */
1850 1354 jae 3f /* take copyop if uaddr > kernelbase */
1851 1355 SMAP_DISABLE_INSTR(3)
1852 1356 jmp do_copy_fault /* Calls leave for us */
1853 1357
1854 1358 _copyout_err:
1855 1359 SMAP_ENABLE_INSTR(4)
1856 1360 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
1857 1361 addq $8, %rsp /* pop bcopy_altentry call ret addr */
1858 1362 3:
1859 1363 movq T_COPYOPS(%r9), %rax
1860 1364 cmpq $0, %rax
1861 1365 jz 2f
1862 1366
1863 1367 /*
1864 1368 * reload args for the copyop
1865 1369 */
1866 1370 movq (%rsp), %rdi
1867 1371 movq 0x8(%rsp), %rsi
|
↓ open down ↓ |
36 lines elided |
↑ open up ↑ |
1868 1372 movq 0x10(%rsp), %rdx
1869 1373 leave
1870 1374 movq CP_COPYOUT(%rax), %rax
1871 1375 INDIRECT_JMP_REG(rax)
1872 1376
1873 1377 2: movl $-1, %eax
1874 1378 leave
1875 1379 ret
1876 1380 SET_SIZE(copyout)
1877 1381
1878 -#elif defined(__i386)
1879 -
1880 -#define ARG_KADDR 4
1881 -#define ARG_UADDR 8
1882 -
1883 - ENTRY(copyout)
1884 - movl kernelbase, %ecx
1885 -#ifdef DEBUG
1886 - cmpl %ecx, ARG_KADDR(%esp)
1887 - jnb 1f
1888 - pushl %ebp
1889 - movl %esp, %ebp
1890 - pushl $.copyout_panic_msg
1891 - call panic
1892 -1:
1893 -#endif
1894 - lea _copyout_err, %eax
1895 - movl %gs:CPU_THREAD, %edx
1896 - cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
1897 - jb do_copy_fault
1898 - jmp 3f
1899 -
1900 -_copyout_err:
1901 - popl %ecx
1902 - popl %edi
1903 - movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
1904 - popl %esi
1905 - popl %ebp
1906 -3:
1907 - movl T_COPYOPS(%edx), %eax
1908 - cmpl $0, %eax
1909 - jz 2f
1910 - jmp *CP_COPYOUT(%eax)
1911 -
1912 -2: movl $-1, %eax
1913 - ret
1914 - SET_SIZE(copyout)
1915 -
1916 -#undef ARG_UADDR
1917 -#undef ARG_KADDR
1918 -
1919 -#endif /* __i386 */
1920 -#endif /* __lint */
1921 -
1922 -#if defined(__lint)
1923 -
1924 -/* ARGSUSED */
1925 -int
1926 -xcopyout_nta(const void *kaddr, void *uaddr, size_t count, int copy_cached)
1927 -{ return (0); }
1928 -
1929 -#else /* __lint */
1930 -
1931 -#if defined(__amd64)
1932 -
1933 1382 ENTRY(xcopyout_nta)
1934 1383 pushq %rbp
1935 1384 movq %rsp, %rbp
1936 1385 subq $24, %rsp
1937 1386
1938 1387 /*
1939 1388 * save args in case we trap and need to rerun as a copyop
1940 1389 */
1941 1390 movq %rdi, (%rsp)
1942 1391 movq %rsi, 0x8(%rsp)
1943 1392 movq %rdx, 0x10(%rsp)
1944 1393
1945 1394 movq kernelbase(%rip), %rax
1946 1395 #ifdef DEBUG
1947 1396 cmpq %rax, %rdi /* %rdi = kaddr */
1948 1397 jnb 1f
1949 1398 leaq .xcopyout_panic_msg(%rip), %rdi
1950 1399 xorl %eax, %eax
1951 1400 call panic
1952 1401 1:
1953 1402 #endif
1954 1403 movq %gs:CPU_THREAD, %r9
1955 1404 cmpq %rax, %rsi /* test uaddr < kernelbase */
1956 1405 jae 4f
1957 1406
1958 1407 cmpq $0, %rcx /* No non-temporal access? */
1959 1408 /*
1960 1409 * pass lofault value as 4th argument to do_copy_fault
1961 1410 */
1962 1411 leaq _xcopyout_err(%rip), %rcx
1963 1412 jnz 6f
1964 1413 /*
1965 1414 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
1966 1415 */
1967 1416 cmpq $XCOPY_MIN_SIZE, %rdx
1968 1417 jae 5f
1969 1418 6:
1970 1419 SMAP_DISABLE_INSTR(4)
1971 1420 jmp do_copy_fault
1972 1421
1973 1422 /*
1974 1423 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
1975 1424 * count is COUNT_ALIGN_SIZE aligned.
1976 1425 */
1977 1426 5:
1978 1427 movq %rdi, %r10
1979 1428 orq %rsi, %r10
1980 1429 andq $NTA_ALIGN_MASK, %r10
1981 1430 orq %rdx, %r10
1982 1431 andq $COUNT_ALIGN_MASK, %r10
1983 1432 jnz 6b
1984 1433 leaq _xcopyout_nta_err(%rip), %rcx
1985 1434 SMAP_DISABLE_INSTR(5)
1986 1435 call do_copy_fault_nta
1987 1436 SMAP_ENABLE_INSTR(5)
1988 1437 ret
1989 1438
1990 1439 4:
1991 1440 movl $EFAULT, %eax
1992 1441 jmp 3f
1993 1442
1994 1443 /*
1995 1444 * A fault during do_copy_fault or do_copy_fault_nta is
1996 1445 * indicated through an errno value in %rax and we iret from the
1997 1446 * trap handler to here.
1998 1447 */
1999 1448 _xcopyout_err:
2000 1449 addq $8, %rsp /* pop bcopy_altentry call ret addr */
2001 1450 _xcopyout_nta_err:
2002 1451 SMAP_ENABLE_INSTR(6)
2003 1452 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
2004 1453 3:
2005 1454 movq T_COPYOPS(%r9), %r8
2006 1455 cmpq $0, %r8
2007 1456 jz 2f
2008 1457
2009 1458 /*
2010 1459 * reload args for the copyop
2011 1460 */
2012 1461 movq (%rsp), %rdi
|
↓ open down ↓ |
70 lines elided |
↑ open up ↑ |
2013 1462 movq 0x8(%rsp), %rsi
2014 1463 movq 0x10(%rsp), %rdx
2015 1464 leave
2016 1465 movq CP_XCOPYOUT(%r8), %r8
2017 1466 INDIRECT_JMP_REG(r8)
2018 1467
2019 1468 2: leave
2020 1469 ret
2021 1470 SET_SIZE(xcopyout_nta)
2022 1471
2023 -#elif defined(__i386)
2024 -
2025 -#define ARG_KADDR 4
2026 -#define ARG_UADDR 8
2027 -#define ARG_COUNT 12
2028 -#define ARG_CACHED 16
2029 -
2030 - ENTRY(xcopyout_nta)
2031 - movl kernelbase, %ecx
2032 - lea _xcopyout_err, %eax
2033 - movl %gs:CPU_THREAD, %edx
2034 - cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
2035 - jae 4f
2036 -
2037 - cmpl $0, use_sse_copy /* no sse support */
2038 - jz do_copy_fault
2039 -
2040 - cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */
2041 - jnz do_copy_fault
2042 -
2043 - /*
2044 - * Make sure cnt is >= XCOPY_MIN_SIZE bytes
2045 - */
2046 - cmpl $XCOPY_MIN_SIZE, %edx
2047 - jb do_copy_fault
2048 -
2049 - /*
2050 - * Make sure src and dst are NTA_ALIGN_SIZE aligned,
2051 - * count is COUNT_ALIGN_SIZE aligned.
2052 - */
2053 - movl ARG_UADDR(%esp), %ecx
2054 - orl ARG_KADDR(%esp), %ecx
2055 - andl $NTA_ALIGN_MASK, %ecx
2056 - orl ARG_COUNT(%esp), %ecx
2057 - andl $COUNT_ALIGN_MASK, %ecx
2058 - jnz do_copy_fault
2059 - jmp do_copy_fault_nta
2060 -
2061 -4:
2062 - movl $EFAULT, %eax
2063 - jmp 3f
2064 -
2065 - /*
2066 - * A fault during do_copy_fault or do_copy_fault_nta is
2067 - * indicated through an errno value in %eax and we iret from the
2068 - * trap handler to here.
2069 - */
2070 -_xcopyout_err:
2071 - / restore the original lofault
2072 - popl %ecx
2073 - popl %edi
2074 - movl %ecx, T_LOFAULT(%edx) / original lofault
2075 - popl %esi
2076 - popl %ebp
2077 -3:
2078 - cmpl $0, T_COPYOPS(%edx)
2079 - jz 2f
2080 - movl T_COPYOPS(%edx), %eax
2081 - jmp *CP_XCOPYOUT(%eax)
2082 -
2083 -2: rep; ret /* use 2 byte return instruction when branch target */
2084 - /* AMD Software Optimization Guide - Section 6.2 */
2085 - SET_SIZE(xcopyout_nta)
2086 -
2087 -#undef ARG_UADDR
2088 -#undef ARG_KADDR
2089 -#undef ARG_COUNT
2090 -#undef ARG_CACHED
2091 -
2092 -#endif /* __i386 */
2093 -#endif /* __lint */
2094 -
2095 1472 /*
2096 1473 * Copy a null terminated string from one point to another in
2097 1474 * the kernel address space.
2098 1475 */
2099 1476
2100 -#if defined(__lint)
2101 -
2102 -/* ARGSUSED */
2103 -int
2104 -copystr(const char *from, char *to, size_t maxlength, size_t *lencopied)
2105 -{ return (0); }
2106 -
2107 -#else /* __lint */
2108 -
2109 -#if defined(__amd64)
2110 -
2111 1477 ENTRY(copystr)
2112 1478 pushq %rbp
2113 1479 movq %rsp, %rbp
2114 1480 #ifdef DEBUG
2115 1481 movq kernelbase(%rip), %rax
2116 1482 cmpq %rax, %rdi /* %rdi = from */
2117 1483 jb 0f
2118 1484 cmpq %rax, %rsi /* %rsi = to */
2119 1485 jnb 1f
2120 1486 0: leaq .copystr_panic_msg(%rip), %rdi
2121 1487 xorl %eax, %eax
2122 1488 call panic
2123 1489 1:
2124 1490 #endif
2125 1491 movq %gs:CPU_THREAD, %r9
2126 1492 movq T_LOFAULT(%r9), %r8 /* pass current lofault value as */
2127 1493 /* 5th argument to do_copystr */
2128 1494 xorl %r10d,%r10d /* pass smap restore need in %r10d */
2129 1495 /* as a non-ABI 6th arg */
2130 1496 do_copystr:
2131 1497 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
2132 1498 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
2133 1499 movq %r8, T_LOFAULT(%r9) /* new lofault */
2134 1500
2135 1501 movq %rdx, %r8 /* save maxlength */
2136 1502
2137 1503 cmpq $0, %rdx /* %rdx = maxlength */
2138 1504 je copystr_enametoolong /* maxlength == 0 */
2139 1505
2140 1506 copystr_loop:
2141 1507 decq %r8
2142 1508 movb (%rdi), %al
2143 1509 incq %rdi
2144 1510 movb %al, (%rsi)
2145 1511 incq %rsi
2146 1512 cmpb $0, %al
2147 1513 je copystr_null /* null char */
2148 1514 cmpq $0, %r8
2149 1515 jne copystr_loop
2150 1516
2151 1517 copystr_enametoolong:
2152 1518 movl $ENAMETOOLONG, %eax
2153 1519 jmp copystr_out
2154 1520
2155 1521 copystr_null:
2156 1522 xorl %eax, %eax /* no error */
2157 1523
2158 1524 copystr_out:
2159 1525 cmpq $0, %rcx /* want length? */
2160 1526 je copystr_smap /* no */
2161 1527 subq %r8, %rdx /* compute length and store it */
2162 1528 movq %rdx, (%rcx)
2163 1529
2164 1530 copystr_smap:
|
↓ open down ↓ |
44 lines elided |
↑ open up ↑ |
2165 1531 cmpl $0, %r10d
2166 1532 jz copystr_done
2167 1533 SMAP_ENABLE_INSTR(7)
2168 1534
2169 1535 copystr_done:
2170 1536 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
2171 1537 leave
2172 1538 ret
2173 1539 SET_SIZE(copystr)
2174 1540
2175 -#elif defined(__i386)
2176 -
2177 -#define ARG_FROM 8
2178 -#define ARG_TO 12
2179 -#define ARG_MAXLEN 16
2180 -#define ARG_LENCOPIED 20
2181 -
2182 - ENTRY(copystr)
2183 -#ifdef DEBUG
2184 - pushl %ebp
2185 - movl %esp, %ebp
2186 - movl kernelbase, %eax
2187 - cmpl %eax, ARG_FROM(%esp)
2188 - jb 0f
2189 - cmpl %eax, ARG_TO(%esp)
2190 - jnb 1f
2191 -0: pushl $.copystr_panic_msg
2192 - call panic
2193 -1: popl %ebp
2194 -#endif
2195 - /* get the current lofault address */
2196 - movl %gs:CPU_THREAD, %eax
2197 - movl T_LOFAULT(%eax), %eax
2198 -do_copystr:
2199 - pushl %ebp /* setup stack frame */
2200 - movl %esp, %ebp
2201 - pushl %ebx /* save registers */
2202 - pushl %edi
2203 -
2204 - movl %gs:CPU_THREAD, %ebx
2205 - movl T_LOFAULT(%ebx), %edi
2206 - pushl %edi /* save the current lofault */
2207 - movl %eax, T_LOFAULT(%ebx) /* new lofault */
2208 -
2209 - movl ARG_MAXLEN(%ebp), %ecx
2210 - cmpl $0, %ecx
2211 - je copystr_enametoolong /* maxlength == 0 */
2212 -
2213 - movl ARG_FROM(%ebp), %ebx /* source address */
2214 - movl ARG_TO(%ebp), %edx /* destination address */
2215 -
2216 -copystr_loop:
2217 - decl %ecx
2218 - movb (%ebx), %al
2219 - incl %ebx
2220 - movb %al, (%edx)
2221 - incl %edx
2222 - cmpb $0, %al
2223 - je copystr_null /* null char */
2224 - cmpl $0, %ecx
2225 - jne copystr_loop
2226 -
2227 -copystr_enametoolong:
2228 - movl $ENAMETOOLONG, %eax
2229 - jmp copystr_out
2230 -
2231 -copystr_null:
2232 - xorl %eax, %eax /* no error */
2233 -
2234 -copystr_out:
2235 - cmpl $0, ARG_LENCOPIED(%ebp) /* want length? */
2236 - je copystr_done /* no */
2237 - movl ARG_MAXLEN(%ebp), %edx
2238 - subl %ecx, %edx /* compute length and store it */
2239 - movl ARG_LENCOPIED(%ebp), %ecx
2240 - movl %edx, (%ecx)
2241 -
2242 -copystr_done:
2243 - popl %edi
2244 - movl %gs:CPU_THREAD, %ebx
2245 - movl %edi, T_LOFAULT(%ebx) /* restore the original lofault */
2246 -
2247 - popl %edi
2248 - popl %ebx
2249 - popl %ebp
2250 - ret
2251 - SET_SIZE(copystr)
2252 -
2253 -#undef ARG_FROM
2254 -#undef ARG_TO
2255 -#undef ARG_MAXLEN
2256 -#undef ARG_LENCOPIED
2257 -
2258 -#endif /* __i386 */
2259 -#endif /* __lint */
2260 -
2261 1541 /*
2262 1542 * Copy a null terminated string from the user address space into
2263 1543 * the kernel address space.
2264 1544 */
2265 1545
2266 -#if defined(__lint)
2267 -
2268 -/* ARGSUSED */
2269 -int
2270 -copyinstr(const char *uaddr, char *kaddr, size_t maxlength,
2271 - size_t *lencopied)
2272 -{ return (0); }
2273 -
2274 -#else /* __lint */
2275 -
2276 -#if defined(__amd64)
2277 -
2278 1546 ENTRY(copyinstr)
2279 1547 pushq %rbp
2280 1548 movq %rsp, %rbp
2281 1549 subq $32, %rsp
2282 1550
2283 1551 /*
2284 1552 * save args in case we trap and need to rerun as a copyop
2285 1553 */
2286 1554 movq %rdi, (%rsp)
2287 1555 movq %rsi, 0x8(%rsp)
2288 1556 movq %rdx, 0x10(%rsp)
2289 1557 movq %rcx, 0x18(%rsp)
2290 1558
2291 1559 movq kernelbase(%rip), %rax
2292 1560 #ifdef DEBUG
2293 1561 cmpq %rax, %rsi /* %rsi = kaddr */
2294 1562 jnb 1f
2295 1563 leaq .copyinstr_panic_msg(%rip), %rdi
2296 1564 xorl %eax, %eax
2297 1565 call panic
2298 1566 1:
2299 1567 #endif
2300 1568 /*
2301 1569 * pass lofault value as 5th argument to do_copystr
2302 1570 * do_copystr expects whether or not we need smap in %r10d
2303 1571 */
2304 1572 leaq _copyinstr_error(%rip), %r8
2305 1573 movl $1, %r10d
2306 1574
2307 1575 cmpq %rax, %rdi /* test uaddr < kernelbase */
2308 1576 jae 4f
2309 1577 SMAP_DISABLE_INSTR(6)
2310 1578 jmp do_copystr
2311 1579 4:
2312 1580 movq %gs:CPU_THREAD, %r9
2313 1581 jmp 3f
2314 1582
2315 1583 _copyinstr_error:
2316 1584 SMAP_ENABLE_INSTR(8)
2317 1585 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
2318 1586 3:
2319 1587 movq T_COPYOPS(%r9), %rax
2320 1588 cmpq $0, %rax
2321 1589 jz 2f
2322 1590
2323 1591 /*
2324 1592 * reload args for the copyop
2325 1593 */
2326 1594 movq (%rsp), %rdi
2327 1595 movq 0x8(%rsp), %rsi
2328 1596 movq 0x10(%rsp), %rdx
|
↓ open down ↓ |
41 lines elided |
↑ open up ↑ |
2329 1597 movq 0x18(%rsp), %rcx
2330 1598 leave
2331 1599 movq CP_COPYINSTR(%rax), %rax
2332 1600 INDIRECT_JMP_REG(rax)
2333 1601
2334 1602 2: movl $EFAULT, %eax /* return EFAULT */
2335 1603 leave
2336 1604 ret
2337 1605 SET_SIZE(copyinstr)
2338 1606
2339 -#elif defined(__i386)
2340 -
2341 -#define ARG_UADDR 4
2342 -#define ARG_KADDR 8
2343 -
2344 - ENTRY(copyinstr)
2345 - movl kernelbase, %ecx
2346 -#ifdef DEBUG
2347 - cmpl %ecx, ARG_KADDR(%esp)
2348 - jnb 1f
2349 - pushl %ebp
2350 - movl %esp, %ebp
2351 - pushl $.copyinstr_panic_msg
2352 - call panic
2353 -1:
2354 -#endif
2355 - lea _copyinstr_error, %eax
2356 - cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
2357 - jb do_copystr
2358 - movl %gs:CPU_THREAD, %edx
2359 - jmp 3f
2360 -
2361 -_copyinstr_error:
2362 - popl %edi
2363 - movl %gs:CPU_THREAD, %edx
2364 - movl %edi, T_LOFAULT(%edx) /* original lofault */
2365 -
2366 - popl %edi
2367 - popl %ebx
2368 - popl %ebp
2369 -3:
2370 - movl T_COPYOPS(%edx), %eax
2371 - cmpl $0, %eax
2372 - jz 2f
2373 - jmp *CP_COPYINSTR(%eax)
2374 -
2375 -2: movl $EFAULT, %eax /* return EFAULT */
2376 - ret
2377 - SET_SIZE(copyinstr)
2378 -
2379 -#undef ARG_UADDR
2380 -#undef ARG_KADDR
2381 -
2382 -#endif /* __i386 */
2383 -#endif /* __lint */
2384 -
2385 1607 /*
2386 1608 * Copy a null terminated string from the kernel
2387 1609 * address space to the user address space.
2388 1610 */
2389 1611
2390 -#if defined(__lint)
2391 -
2392 -/* ARGSUSED */
2393 -int
2394 -copyoutstr(const char *kaddr, char *uaddr, size_t maxlength,
2395 - size_t *lencopied)
2396 -{ return (0); }
2397 -
2398 -#else /* __lint */
2399 -
2400 -#if defined(__amd64)
2401 -
2402 1612 ENTRY(copyoutstr)
2403 1613 pushq %rbp
2404 1614 movq %rsp, %rbp
2405 1615 subq $32, %rsp
2406 1616
2407 1617 /*
2408 1618 * save args in case we trap and need to rerun as a copyop
2409 1619 */
2410 1620 movq %rdi, (%rsp)
2411 1621 movq %rsi, 0x8(%rsp)
2412 1622 movq %rdx, 0x10(%rsp)
2413 1623 movq %rcx, 0x18(%rsp)
2414 1624
2415 1625 movq kernelbase(%rip), %rax
2416 1626 #ifdef DEBUG
2417 1627 cmpq %rax, %rdi /* %rdi = kaddr */
2418 1628 jnb 1f
2419 1629 leaq .copyoutstr_panic_msg(%rip), %rdi
2420 1630 jmp call_panic /* setup stack and call panic */
2421 1631 1:
2422 1632 #endif
2423 1633 /*
2424 1634 * pass lofault value as 5th argument to do_copystr
2425 1635 * pass one as 6th argument to do_copystr in %r10d
2426 1636 */
2427 1637 leaq _copyoutstr_error(%rip), %r8
2428 1638 movl $1, %r10d
2429 1639
2430 1640 cmpq %rax, %rsi /* test uaddr < kernelbase */
2431 1641 jae 4f
2432 1642 SMAP_DISABLE_INSTR(7)
2433 1643 jmp do_copystr
2434 1644 4:
2435 1645 movq %gs:CPU_THREAD, %r9
2436 1646 jmp 3f
2437 1647
2438 1648 _copyoutstr_error:
2439 1649 SMAP_ENABLE_INSTR(9)
2440 1650 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
2441 1651 3:
2442 1652 movq T_COPYOPS(%r9), %rax
2443 1653 cmpq $0, %rax
2444 1654 jz 2f
2445 1655
2446 1656 /*
2447 1657 * reload args for the copyop
2448 1658 */
2449 1659 movq (%rsp), %rdi
2450 1660 movq 0x8(%rsp), %rsi
2451 1661 movq 0x10(%rsp), %rdx
|
↓ open down ↓ |
40 lines elided |
↑ open up ↑ |
2452 1662 movq 0x18(%rsp), %rcx
2453 1663 leave
2454 1664 movq CP_COPYOUTSTR(%rax), %rax
2455 1665 INDIRECT_JMP_REG(rax)
2456 1666
2457 1667 2: movl $EFAULT, %eax /* return EFAULT */
2458 1668 leave
2459 1669 ret
2460 1670 SET_SIZE(copyoutstr)
2461 1671
2462 -#elif defined(__i386)
2463 -
2464 -#define ARG_KADDR 4
2465 -#define ARG_UADDR 8
2466 -
2467 - ENTRY(copyoutstr)
2468 - movl kernelbase, %ecx
2469 -#ifdef DEBUG
2470 - cmpl %ecx, ARG_KADDR(%esp)
2471 - jnb 1f
2472 - pushl %ebp
2473 - movl %esp, %ebp
2474 - pushl $.copyoutstr_panic_msg
2475 - call panic
2476 -1:
2477 -#endif
2478 - lea _copyoutstr_error, %eax
2479 - cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
2480 - jb do_copystr
2481 - movl %gs:CPU_THREAD, %edx
2482 - jmp 3f
2483 -
2484 -_copyoutstr_error:
2485 - popl %edi
2486 - movl %gs:CPU_THREAD, %edx
2487 - movl %edi, T_LOFAULT(%edx) /* restore the original lofault */
2488 -
2489 - popl %edi
2490 - popl %ebx
2491 - popl %ebp
2492 -3:
2493 - movl T_COPYOPS(%edx), %eax
2494 - cmpl $0, %eax
2495 - jz 2f
2496 - jmp *CP_COPYOUTSTR(%eax)
2497 -
2498 -2: movl $EFAULT, %eax /* return EFAULT */
2499 - ret
2500 - SET_SIZE(copyoutstr)
2501 -
2502 -#undef ARG_KADDR
2503 -#undef ARG_UADDR
2504 -
2505 -#endif /* __i386 */
2506 -#endif /* __lint */
2507 -
2508 1672 /*
2509 1673 * Since all of the fuword() variants are so similar, we have a macro to spit
2510 1674 * them out. This allows us to create DTrace-unobservable functions easily.
2511 1675 */
2512 1676
2513 -#if defined(__lint)
2514 -
2515 -#if defined(__amd64)
2516 -
2517 -/* ARGSUSED */
2518 -int
2519 -fuword64(const void *addr, uint64_t *dst)
2520 -{ return (0); }
2521 -
2522 -#endif
2523 -
2524 -/* ARGSUSED */
2525 -int
2526 -fuword32(const void *addr, uint32_t *dst)
2527 -{ return (0); }
2528 -
2529 -/* ARGSUSED */
2530 -int
2531 -fuword16(const void *addr, uint16_t *dst)
2532 -{ return (0); }
2533 -
2534 -/* ARGSUSED */
2535 -int
2536 -fuword8(const void *addr, uint8_t *dst)
2537 -{ return (0); }
2538 -
2539 -#else /* __lint */
2540 -
2541 -#if defined(__amd64)
2542 -
2543 1677 /*
2544 1678 * Note that we don't save and reload the arguments here
2545 1679 * because their values are not altered in the copy path.
2546 1680 * Additionally, when successful, the smap_enable jmp will
2547 1681 * actually return us to our original caller.
2548 1682 */
2549 1683
2550 1684 #define FUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2) \
2551 1685 ENTRY(NAME) \
2552 1686 movq %gs:CPU_THREAD, %r9; \
2553 1687 cmpq kernelbase(%rip), %rdi; \
2554 1688 jae 1f; \
2555 1689 leaq _flt_/**/NAME, %rdx; \
2556 1690 movq %rdx, T_LOFAULT(%r9); \
2557 1691 SMAP_DISABLE_INSTR(DISNUM) \
2558 1692 INSTR (%rdi), REG; \
2559 1693 movq $0, T_LOFAULT(%r9); \
2560 1694 INSTR REG, (%rsi); \
2561 1695 xorl %eax, %eax; \
2562 1696 SMAP_ENABLE_INSTR(EN1) \
2563 1697 ret; \
2564 1698 _flt_/**/NAME: \
2565 1699 SMAP_ENABLE_INSTR(EN2) \
2566 1700 movq $0, T_LOFAULT(%r9); \
2567 1701 1: \
2568 1702 movq T_COPYOPS(%r9), %rax; \
2569 1703 cmpq $0, %rax; \
2570 1704 jz 2f; \
2571 1705 movq COPYOP(%rax), %rax; \
2572 1706 INDIRECT_JMP_REG(rax); \
|
↓ open down ↓ |
20 lines elided |
↑ open up ↑ |
2573 1707 2: \
2574 1708 movl $-1, %eax; \
2575 1709 ret; \
2576 1710 SET_SIZE(NAME)
2577 1711
2578 1712 FUWORD(fuword64, movq, %rax, CP_FUWORD64,8,10,11)
2579 1713 FUWORD(fuword32, movl, %eax, CP_FUWORD32,9,12,13)
2580 1714 FUWORD(fuword16, movw, %ax, CP_FUWORD16,10,14,15)
2581 1715 FUWORD(fuword8, movb, %al, CP_FUWORD8,11,16,17)
2582 1716
2583 -#elif defined(__i386)
2584 -
2585 -#define FUWORD(NAME, INSTR, REG, COPYOP) \
2586 - ENTRY(NAME) \
2587 - movl %gs:CPU_THREAD, %ecx; \
2588 - movl kernelbase, %eax; \
2589 - cmpl %eax, 4(%esp); \
2590 - jae 1f; \
2591 - lea _flt_/**/NAME, %edx; \
2592 - movl %edx, T_LOFAULT(%ecx); \
2593 - movl 4(%esp), %eax; \
2594 - movl 8(%esp), %edx; \
2595 - INSTR (%eax), REG; \
2596 - movl $0, T_LOFAULT(%ecx); \
2597 - INSTR REG, (%edx); \
2598 - xorl %eax, %eax; \
2599 - ret; \
2600 -_flt_/**/NAME: \
2601 - movl $0, T_LOFAULT(%ecx); \
2602 -1: \
2603 - movl T_COPYOPS(%ecx), %eax; \
2604 - cmpl $0, %eax; \
2605 - jz 2f; \
2606 - jmp *COPYOP(%eax); \
2607 -2: \
2608 - movl $-1, %eax; \
2609 - ret; \
2610 - SET_SIZE(NAME)
2611 -
2612 - FUWORD(fuword32, movl, %eax, CP_FUWORD32)
2613 - FUWORD(fuword16, movw, %ax, CP_FUWORD16)
2614 - FUWORD(fuword8, movb, %al, CP_FUWORD8)
2615 -
2616 -#endif /* __i386 */
2617 -
2618 1717 #undef FUWORD
2619 1718
2620 -#endif /* __lint */
2621 -
2622 1719 /*
2623 1720 * Set user word.
2624 1721 */
2625 1722
2626 -#if defined(__lint)
2627 -
2628 -#if defined(__amd64)
2629 -
2630 -/* ARGSUSED */
2631 -int
2632 -suword64(void *addr, uint64_t value)
2633 -{ return (0); }
2634 -
2635 -#endif
2636 -
2637 -/* ARGSUSED */
2638 -int
2639 -suword32(void *addr, uint32_t value)
2640 -{ return (0); }
2641 -
2642 -/* ARGSUSED */
2643 -int
2644 -suword16(void *addr, uint16_t value)
2645 -{ return (0); }
2646 -
2647 -/* ARGSUSED */
2648 -int
2649 -suword8(void *addr, uint8_t value)
2650 -{ return (0); }
2651 -
2652 -#else /* lint */
2653 -
2654 -#if defined(__amd64)
2655 -
2656 1723 /*
2657 1724 * Note that we don't save and reload the arguments here
2658 1725 * because their values are not altered in the copy path.
2659 1726 */
2660 1727
2661 1728 #define SUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2) \
2662 1729 ENTRY(NAME) \
2663 1730 movq %gs:CPU_THREAD, %r9; \
2664 1731 cmpq kernelbase(%rip), %rdi; \
2665 1732 jae 1f; \
2666 1733 leaq _flt_/**/NAME, %rdx; \
2667 1734 SMAP_DISABLE_INSTR(DISNUM) \
2668 1735 movq %rdx, T_LOFAULT(%r9); \
2669 1736 INSTR REG, (%rdi); \
2670 1737 movq $0, T_LOFAULT(%r9); \
2671 1738 xorl %eax, %eax; \
2672 1739 SMAP_ENABLE_INSTR(EN1) \
2673 1740 ret; \
2674 1741 _flt_/**/NAME: \
2675 1742 SMAP_ENABLE_INSTR(EN2) \
2676 1743 movq $0, T_LOFAULT(%r9); \
2677 1744 1: \
2678 1745 movq T_COPYOPS(%r9), %rax; \
2679 1746 cmpq $0, %rax; \
2680 1747 jz 3f; \
2681 1748 movq COPYOP(%rax), %rax; \
2682 1749 INDIRECT_JMP_REG(rax); \
|
↓ open down ↓ |
17 lines elided |
↑ open up ↑ |
2683 1750 3: \
2684 1751 movl $-1, %eax; \
2685 1752 ret; \
2686 1753 SET_SIZE(NAME)
2687 1754
2688 1755 SUWORD(suword64, movq, %rsi, CP_SUWORD64,12,18,19)
2689 1756 SUWORD(suword32, movl, %esi, CP_SUWORD32,13,20,21)
2690 1757 SUWORD(suword16, movw, %si, CP_SUWORD16,14,22,23)
2691 1758 SUWORD(suword8, movb, %sil, CP_SUWORD8,15,24,25)
2692 1759
2693 -#elif defined(__i386)
2694 -
2695 -#define SUWORD(NAME, INSTR, REG, COPYOP) \
2696 - ENTRY(NAME) \
2697 - movl %gs:CPU_THREAD, %ecx; \
2698 - movl kernelbase, %eax; \
2699 - cmpl %eax, 4(%esp); \
2700 - jae 1f; \
2701 - lea _flt_/**/NAME, %edx; \
2702 - movl %edx, T_LOFAULT(%ecx); \
2703 - movl 4(%esp), %eax; \
2704 - movl 8(%esp), %edx; \
2705 - INSTR REG, (%eax); \
2706 - movl $0, T_LOFAULT(%ecx); \
2707 - xorl %eax, %eax; \
2708 - ret; \
2709 -_flt_/**/NAME: \
2710 - movl $0, T_LOFAULT(%ecx); \
2711 -1: \
2712 - movl T_COPYOPS(%ecx), %eax; \
2713 - cmpl $0, %eax; \
2714 - jz 3f; \
2715 - movl COPYOP(%eax), %ecx; \
2716 - jmp *%ecx; \
2717 -3: \
2718 - movl $-1, %eax; \
2719 - ret; \
2720 - SET_SIZE(NAME)
2721 -
2722 - SUWORD(suword32, movl, %edx, CP_SUWORD32)
2723 - SUWORD(suword16, movw, %dx, CP_SUWORD16)
2724 - SUWORD(suword8, movb, %dl, CP_SUWORD8)
2725 -
2726 -#endif /* __i386 */
2727 -
2728 1760 #undef SUWORD
2729 1761
2730 -#endif /* __lint */
2731 -
2732 -#if defined(__lint)
2733 -
2734 -#if defined(__amd64)
2735 -
2736 -/*ARGSUSED*/
2737 -void
2738 -fuword64_noerr(const void *addr, uint64_t *dst)
2739 -{}
2740 -
2741 -#endif
2742 -
2743 -/*ARGSUSED*/
2744 -void
2745 -fuword32_noerr(const void *addr, uint32_t *dst)
2746 -{}
2747 -
2748 -/*ARGSUSED*/
2749 -void
2750 -fuword8_noerr(const void *addr, uint8_t *dst)
2751 -{}
2752 -
2753 -/*ARGSUSED*/
2754 -void
2755 -fuword16_noerr(const void *addr, uint16_t *dst)
2756 -{}
2757 -
2758 -#else /* __lint */
2759 -
2760 -#if defined(__amd64)
2761 -
2762 1762 #define FUWORD_NOERR(NAME, INSTR, REG) \
2763 1763 ENTRY(NAME) \
2764 1764 cmpq kernelbase(%rip), %rdi; \
2765 1765 cmovnbq kernelbase(%rip), %rdi; \
2766 1766 INSTR (%rdi), REG; \
2767 1767 INSTR REG, (%rsi); \
2768 1768 ret; \
2769 1769 SET_SIZE(NAME)
2770 1770
2771 1771 FUWORD_NOERR(fuword64_noerr, movq, %rax)
2772 1772 FUWORD_NOERR(fuword32_noerr, movl, %eax)
2773 1773 FUWORD_NOERR(fuword16_noerr, movw, %ax)
2774 1774 FUWORD_NOERR(fuword8_noerr, movb, %al)
2775 1775
2776 -#elif defined(__i386)
2777 -
2778 -#define FUWORD_NOERR(NAME, INSTR, REG) \
2779 - ENTRY(NAME) \
2780 - movl 4(%esp), %eax; \
2781 - cmpl kernelbase, %eax; \
2782 - jb 1f; \
2783 - movl kernelbase, %eax; \
2784 -1: movl 8(%esp), %edx; \
2785 - INSTR (%eax), REG; \
2786 - INSTR REG, (%edx); \
2787 - ret; \
2788 - SET_SIZE(NAME)
2789 -
2790 - FUWORD_NOERR(fuword32_noerr, movl, %ecx)
2791 - FUWORD_NOERR(fuword16_noerr, movw, %cx)
2792 - FUWORD_NOERR(fuword8_noerr, movb, %cl)
2793 -
2794 -#endif /* __i386 */
2795 -
2796 1776 #undef FUWORD_NOERR
2797 1777
2798 -#endif /* __lint */
2799 -
2800 -#if defined(__lint)
2801 -
2802 -#if defined(__amd64)
2803 -
2804 -/*ARGSUSED*/
2805 -void
2806 -suword64_noerr(void *addr, uint64_t value)
2807 -{}
2808 -
2809 -#endif
2810 -
2811 -/*ARGSUSED*/
2812 -void
2813 -suword32_noerr(void *addr, uint32_t value)
2814 -{}
2815 -
2816 -/*ARGSUSED*/
2817 -void
2818 -suword16_noerr(void *addr, uint16_t value)
2819 -{}
2820 -
2821 -/*ARGSUSED*/
2822 -void
2823 -suword8_noerr(void *addr, uint8_t value)
2824 -{}
2825 -
2826 -#else /* lint */
2827 -
2828 -#if defined(__amd64)
2829 -
2830 1778 #define SUWORD_NOERR(NAME, INSTR, REG) \
2831 1779 ENTRY(NAME) \
2832 1780 cmpq kernelbase(%rip), %rdi; \
2833 1781 cmovnbq kernelbase(%rip), %rdi; \
2834 1782 INSTR REG, (%rdi); \
2835 1783 ret; \
2836 1784 SET_SIZE(NAME)
2837 1785
2838 1786 SUWORD_NOERR(suword64_noerr, movq, %rsi)
2839 1787 SUWORD_NOERR(suword32_noerr, movl, %esi)
2840 1788 SUWORD_NOERR(suword16_noerr, movw, %si)
2841 1789 SUWORD_NOERR(suword8_noerr, movb, %sil)
2842 1790
2843 -#elif defined(__i386)
2844 -
2845 -#define SUWORD_NOERR(NAME, INSTR, REG) \
2846 - ENTRY(NAME) \
2847 - movl 4(%esp), %eax; \
2848 - cmpl kernelbase, %eax; \
2849 - jb 1f; \
2850 - movl kernelbase, %eax; \
2851 -1: \
2852 - movl 8(%esp), %edx; \
2853 - INSTR REG, (%eax); \
2854 - ret; \
2855 - SET_SIZE(NAME)
2856 -
2857 - SUWORD_NOERR(suword32_noerr, movl, %edx)
2858 - SUWORD_NOERR(suword16_noerr, movw, %dx)
2859 - SUWORD_NOERR(suword8_noerr, movb, %dl)
2860 -
2861 -#endif /* __i386 */
2862 -
2863 1791 #undef SUWORD_NOERR
2864 1792
2865 -#endif /* lint */
2866 1793
2867 -
2868 -#if defined(__lint)
2869 -
2870 -/*ARGSUSED*/
2871 -int
2872 -subyte(void *addr, uchar_t value)
2873 -{ return (0); }
2874 -
2875 -/*ARGSUSED*/
2876 -void
2877 -subyte_noerr(void *addr, uchar_t value)
2878 -{}
2879 -
2880 -/*ARGSUSED*/
2881 -int
2882 -fulword(const void *addr, ulong_t *valuep)
2883 -{ return (0); }
2884 -
2885 -/*ARGSUSED*/
2886 -void
2887 -fulword_noerr(const void *addr, ulong_t *valuep)
2888 -{}
2889 -
2890 -/*ARGSUSED*/
2891 -int
2892 -sulword(void *addr, ulong_t valuep)
2893 -{ return (0); }
2894 -
2895 -/*ARGSUSED*/
2896 -void
2897 -sulword_noerr(void *addr, ulong_t valuep)
2898 -{}
2899 -
2900 -#else
2901 -
2902 1794 .weak subyte
2903 1795 subyte=suword8
2904 1796 .weak subyte_noerr
2905 1797 subyte_noerr=suword8_noerr
2906 1798
2907 -#if defined(__amd64)
2908 -
2909 1799 .weak fulword
2910 1800 fulword=fuword64
2911 1801 .weak fulword_noerr
2912 1802 fulword_noerr=fuword64_noerr
2913 1803 .weak sulword
2914 1804 sulword=suword64
2915 1805 .weak sulword_noerr
2916 1806 sulword_noerr=suword64_noerr
2917 1807
2918 -#elif defined(__i386)
2919 -
2920 - .weak fulword
2921 - fulword=fuword32
2922 - .weak fulword_noerr
2923 - fulword_noerr=fuword32_noerr
2924 - .weak sulword
2925 - sulword=suword32
2926 - .weak sulword_noerr
2927 - sulword_noerr=suword32_noerr
2928 -
2929 -#endif /* __i386 */
2930 -
2931 -#endif /* __lint */
2932 -
2933 -#if defined(__lint)
2934 -
2935 -/*
2936 - * Copy a block of storage - must not overlap (from + len <= to).
2937 - * No fault handler installed (to be called under on_fault())
2938 - */
2939 -
2940 -/* ARGSUSED */
2941 -void
2942 -copyout_noerr(const void *kfrom, void *uto, size_t count)
2943 -{}
2944 -
2945 -/* ARGSUSED */
2946 -void
2947 -copyin_noerr(const void *ufrom, void *kto, size_t count)
2948 -{}
2949 -
2950 -/*
2951 - * Zero a block of storage in user space
2952 - */
2953 -
2954 -/* ARGSUSED */
2955 -void
2956 -uzero(void *addr, size_t count)
2957 -{}
2958 -
2959 -/*
2960 - * copy a block of storage in user space
2961 - */
2962 -
2963 -/* ARGSUSED */
2964 -void
2965 -ucopy(const void *ufrom, void *uto, size_t ulength)
2966 -{}
2967 -
2968 -/*
2969 - * copy a string in user space
2970 - */
2971 -
2972 -/* ARGSUSED */
2973 -void
2974 -ucopystr(const char *ufrom, char *uto, size_t umaxlength, size_t *lencopied)
2975 -{}
2976 -
2977 -#else /* __lint */
2978 -
2979 -#if defined(__amd64)
2980 -
2981 1808 ENTRY(copyin_noerr)
2982 1809 movq kernelbase(%rip), %rax
2983 1810 #ifdef DEBUG
2984 1811 cmpq %rax, %rsi /* %rsi = kto */
2985 1812 jae 1f
2986 1813 leaq .cpyin_ne_pmsg(%rip), %rdi
2987 1814 jmp call_panic /* setup stack and call panic */
2988 1815 1:
2989 1816 #endif
2990 1817 cmpq %rax, %rdi /* ufrom < kernelbase */
2991 1818 jb do_copy
2992 1819 movq %rax, %rdi /* force fault at kernelbase */
2993 1820 jmp do_copy
2994 1821 SET_SIZE(copyin_noerr)
2995 1822
2996 1823 ENTRY(copyout_noerr)
2997 1824 movq kernelbase(%rip), %rax
2998 1825 #ifdef DEBUG
2999 1826 cmpq %rax, %rdi /* %rdi = kfrom */
3000 1827 jae 1f
3001 1828 leaq .cpyout_ne_pmsg(%rip), %rdi
3002 1829 jmp call_panic /* setup stack and call panic */
3003 1830 1:
3004 1831 #endif
3005 1832 cmpq %rax, %rsi /* uto < kernelbase */
3006 1833 jb do_copy
3007 1834 movq %rax, %rsi /* force fault at kernelbase */
3008 1835 jmp do_copy
3009 1836 SET_SIZE(copyout_noerr)
3010 1837
3011 1838 ENTRY(uzero)
3012 1839 movq kernelbase(%rip), %rax
3013 1840 cmpq %rax, %rdi
3014 1841 jb do_zero
3015 1842 movq %rax, %rdi /* force fault at kernelbase */
3016 1843 jmp do_zero
3017 1844 SET_SIZE(uzero)
3018 1845
3019 1846 ENTRY(ucopy)
3020 1847 movq kernelbase(%rip), %rax
3021 1848 cmpq %rax, %rdi
3022 1849 cmovaeq %rax, %rdi /* force fault at kernelbase */
3023 1850 cmpq %rax, %rsi
3024 1851 cmovaeq %rax, %rsi /* force fault at kernelbase */
3025 1852 jmp do_copy
3026 1853 SET_SIZE(ucopy)
3027 1854
3028 1855 /*
3029 1856 * Note, the frame pointer is required here becuase do_copystr expects
3030 1857 * to be able to pop it off!
3031 1858 */
3032 1859 ENTRY(ucopystr)
3033 1860 pushq %rbp
3034 1861 movq %rsp, %rbp
3035 1862 movq kernelbase(%rip), %rax
3036 1863 cmpq %rax, %rdi
3037 1864 cmovaeq %rax, %rdi /* force fault at kernelbase */
|
↓ open down ↓ |
47 lines elided |
↑ open up ↑ |
3038 1865 cmpq %rax, %rsi
3039 1866 cmovaeq %rax, %rsi /* force fault at kernelbase */
3040 1867 /* do_copystr expects lofault address in %r8 */
3041 1868 /* do_copystr expects whether or not we need smap in %r10 */
3042 1869 xorl %r10d, %r10d
3043 1870 movq %gs:CPU_THREAD, %r8
3044 1871 movq T_LOFAULT(%r8), %r8
3045 1872 jmp do_copystr
3046 1873 SET_SIZE(ucopystr)
3047 1874
3048 -#elif defined(__i386)
3049 -
3050 - ENTRY(copyin_noerr)
3051 - movl kernelbase, %eax
3052 1875 #ifdef DEBUG
3053 - cmpl %eax, 8(%esp)
3054 - jae 1f
3055 - pushl $.cpyin_ne_pmsg
3056 - call panic
3057 -1:
3058 -#endif
3059 - cmpl %eax, 4(%esp)
3060 - jb do_copy
3061 - movl %eax, 4(%esp) /* force fault at kernelbase */
3062 - jmp do_copy
3063 - SET_SIZE(copyin_noerr)
3064 -
3065 - ENTRY(copyout_noerr)
3066 - movl kernelbase, %eax
3067 -#ifdef DEBUG
3068 - cmpl %eax, 4(%esp)
3069 - jae 1f
3070 - pushl $.cpyout_ne_pmsg
3071 - call panic
3072 -1:
3073 -#endif
3074 - cmpl %eax, 8(%esp)
3075 - jb do_copy
3076 - movl %eax, 8(%esp) /* force fault at kernelbase */
3077 - jmp do_copy
3078 - SET_SIZE(copyout_noerr)
3079 -
3080 - ENTRY(uzero)
3081 - movl kernelbase, %eax
3082 - cmpl %eax, 4(%esp)
3083 - jb do_zero
3084 - movl %eax, 4(%esp) /* force fault at kernelbase */
3085 - jmp do_zero
3086 - SET_SIZE(uzero)
3087 -
3088 - ENTRY(ucopy)
3089 - movl kernelbase, %eax
3090 - cmpl %eax, 4(%esp)
3091 - jb 1f
3092 - movl %eax, 4(%esp) /* force fault at kernelbase */
3093 -1:
3094 - cmpl %eax, 8(%esp)
3095 - jb do_copy
3096 - movl %eax, 8(%esp) /* force fault at kernelbase */
3097 - jmp do_copy
3098 - SET_SIZE(ucopy)
3099 -
3100 - ENTRY(ucopystr)
3101 - movl kernelbase, %eax
3102 - cmpl %eax, 4(%esp)
3103 - jb 1f
3104 - movl %eax, 4(%esp) /* force fault at kernelbase */
3105 -1:
3106 - cmpl %eax, 8(%esp)
3107 - jb 2f
3108 - movl %eax, 8(%esp) /* force fault at kernelbase */
3109 -2:
3110 - /* do_copystr expects the lofault address in %eax */
3111 - movl %gs:CPU_THREAD, %eax
3112 - movl T_LOFAULT(%eax), %eax
3113 - jmp do_copystr
3114 - SET_SIZE(ucopystr)
3115 -
3116 -#endif /* __i386 */
3117 -
3118 -#ifdef DEBUG
3119 1876 .data
3120 1877 .kcopy_panic_msg:
3121 1878 .string "kcopy: arguments below kernelbase"
3122 1879 .bcopy_panic_msg:
3123 1880 .string "bcopy: arguments below kernelbase"
3124 1881 .kzero_panic_msg:
3125 1882 .string "kzero: arguments below kernelbase"
3126 1883 .bzero_panic_msg:
3127 1884 .string "bzero: arguments below kernelbase"
3128 1885 .copyin_panic_msg:
3129 1886 .string "copyin: kaddr argument below kernelbase"
3130 1887 .xcopyin_panic_msg:
3131 1888 .string "xcopyin: kaddr argument below kernelbase"
3132 1889 .copyout_panic_msg:
3133 1890 .string "copyout: kaddr argument below kernelbase"
3134 1891 .xcopyout_panic_msg:
3135 1892 .string "xcopyout: kaddr argument below kernelbase"
3136 1893 .copystr_panic_msg:
3137 1894 .string "copystr: arguments in user space"
|
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
3138 1895 .copyinstr_panic_msg:
3139 1896 .string "copyinstr: kaddr argument not in kernel address space"
3140 1897 .copyoutstr_panic_msg:
3141 1898 .string "copyoutstr: kaddr argument not in kernel address space"
3142 1899 .cpyin_ne_pmsg:
3143 1900 .string "copyin_noerr: argument not in kernel address space"
3144 1901 .cpyout_ne_pmsg:
3145 1902 .string "copyout_noerr: argument not in kernel address space"
3146 1903 #endif
3147 1904
3148 -#endif /* __lint */
3149 -
3150 1905 /*
3151 1906 * These functions are used for SMAP, supervisor mode access protection. They
3152 1907 * are hotpatched to become real instructions when the system starts up which is
3153 1908 * done in mlsetup() as a part of enabling the other CR4 related features.
3154 1909 *
3155 1910 * Generally speaking, smap_disable() is a stac instruction and smap_enable is a
3156 1911 * clac instruction. It's safe to call these any number of times, and in fact,
3157 1912 * out of paranoia, the kernel will likely call it at several points.
3158 1913 */
3159 1914
3160 -#if defined(__lint)
3161 -
3162 -void
3163 -smap_enable(void)
3164 -{}
3165 -
3166 -void
3167 -smap_disable(void)
3168 -{}
3169 -
3170 -#else
3171 -
3172 -#if defined (__amd64) || defined(__i386)
3173 1915 ENTRY(smap_disable)
3174 1916 nop
3175 1917 nop
3176 1918 nop
3177 1919 ret
3178 1920 SET_SIZE(smap_disable)
3179 1921
3180 1922 ENTRY(smap_enable)
3181 1923 nop
3182 1924 nop
3183 1925 nop
3184 1926 ret
3185 1927 SET_SIZE(smap_enable)
3186 1928
3187 -#endif /* __amd64 || __i386 */
3188 -
3189 -#endif /* __lint */
3190 -
3191 -#ifndef __lint
3192 -
3193 1929 .data
3194 1930 .align 4
3195 1931 .globl _smap_enable_patch_count
3196 1932 .type _smap_enable_patch_count,@object
3197 1933 .size _smap_enable_patch_count, 4
3198 1934 _smap_enable_patch_count:
3199 1935 .long SMAP_ENABLE_COUNT
3200 1936
3201 1937 .globl _smap_disable_patch_count
3202 1938 .type _smap_disable_patch_count,@object
3203 1939 .size _smap_disable_patch_count, 4
3204 1940 _smap_disable_patch_count:
3205 1941 .long SMAP_DISABLE_COUNT
3206 -
3207 -#endif /* __lint */
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX