25
26 /*
27 * Copyright (c) 2009, Intel Corporation
28 * All rights reserved.
29 */
30
31 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
32 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
33 /* All Rights Reserved */
34
35 /* Copyright (c) 1987, 1988 Microsoft Corporation */
36 /* All Rights Reserved */
37
38 /*
39 * Copyright 2019 Joyent, Inc.
40 */
41
42 #include <sys/errno.h>
43 #include <sys/asm_linkage.h>
44
45 #if defined(__lint)
46 #include <sys/types.h>
47 #include <sys/systm.h>
48 #else /* __lint */
49 #include "assym.h"
50 #endif /* __lint */
51
52 #define KCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
53 #define XCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
54 /*
55 * Non-temopral access (NTA) alignment requirement
56 */
57 #define NTA_ALIGN_SIZE 4 /* Must be at least 4-byte aligned */
58 #define NTA_ALIGN_MASK _CONST(NTA_ALIGN_SIZE-1)
59 #define COUNT_ALIGN_SIZE 16 /* Must be at least 16-byte aligned */
60 #define COUNT_ALIGN_MASK _CONST(COUNT_ALIGN_SIZE-1)
61
62 /*
63 * With the introduction of Broadwell, Intel has introduced supervisor mode
64 * access protection -- SMAP. SMAP forces the kernel to set certain bits to
65 * enable access of user pages (AC in rflags, defines as PS_ACHK in
66 * <sys/psw.h>). One of the challenges is that the implementation of many of the
67 * userland copy routines directly use the kernel ones. For example, copyin and
68 * copyout simply go and jump to the do_copy_fault label and traditionally let
69 * those deal with the return for them. In fact, changing that is a can of frame
70 * pointers.
126 * The optimal 64-bit bcopy and kcopy for modern x86 processors uses
127 * "rep smovq" for large sizes. Performance data shows that many calls to
128 * bcopy/kcopy/bzero/kzero operate on small buffers. For best performance for
129 * these small sizes unrolled code is used. For medium sizes loops writing
130 * 64-bytes per loop are used. Transition points were determined experimentally.
131 */
132 #define BZERO_USE_REP (1024)
133 #define BCOPY_DFLT_REP (128)
134 #define BCOPY_NHM_REP (768)
135
136 /*
137 * Copy a block of storage, returning an error code if `from' or
138 * `to' takes a kernel pagefault which cannot be resolved.
139 * Returns errno value on pagefault error, 0 if all ok
140 */
141
142 /*
143 * I'm sorry about these macros, but copy.s is unsurprisingly sensitive to
144 * additional call instructions.
145 */
146 #if defined(__amd64)
147 #define SMAP_DISABLE_COUNT 16
148 #define SMAP_ENABLE_COUNT 26
149 #elif defined(__i386)
150 #define SMAP_DISABLE_COUNT 0
151 #define SMAP_ENABLE_COUNT 0
152 #endif
153
154 #define SMAP_DISABLE_INSTR(ITER) \
155 .globl _smap_disable_patch_/**/ITER; \
156 _smap_disable_patch_/**/ITER/**/:; \
157 nop; nop; nop;
158
159 #define SMAP_ENABLE_INSTR(ITER) \
160 .globl _smap_enable_patch_/**/ITER; \
161 _smap_enable_patch_/**/ITER/**/:; \
162 nop; nop; nop;
163
164 #if defined(__lint)
165
166 /* ARGSUSED */
167 int
168 kcopy(const void *from, void *to, size_t count)
169 { return (0); }
170
171 #else /* __lint */
172
173 .globl kernelbase
174 .globl postbootkernelbase
175
176 #if defined(__amd64)
177
178 ENTRY(kcopy)
179 pushq %rbp
180 movq %rsp, %rbp
181 #ifdef DEBUG
182 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
183 jb 0f
184 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
185 jnb 1f
186 0: leaq .kcopy_panic_msg(%rip), %rdi
187 xorl %eax, %eax
188 call panic
189 1:
190 #endif
191 /*
192 * pass lofault value as 4th argument to do_copy_fault
193 */
194 leaq _kcopy_copyerr(%rip), %rcx
195 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
196
197 do_copy_fault:
198 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
199 movq %rcx, T_LOFAULT(%r9) /* new lofault */
200 call bcopy_altentry
201 xorl %eax, %eax /* return 0 (success) */
202 SMAP_ENABLE_INSTR(0)
203
204 /*
205 * A fault during do_copy_fault is indicated through an errno value
206 * in %rax and we iretq from the trap handler to here.
207 */
208 _kcopy_copyerr:
209 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
210 leave
211 ret
212 SET_SIZE(kcopy)
213
214 #elif defined(__i386)
215
216 #define ARG_FROM 8
217 #define ARG_TO 12
218 #define ARG_COUNT 16
219
220 ENTRY(kcopy)
221 #ifdef DEBUG
222 pushl %ebp
223 movl %esp, %ebp
224 movl postbootkernelbase, %eax
225 cmpl %eax, ARG_FROM(%ebp)
226 jb 0f
227 cmpl %eax, ARG_TO(%ebp)
228 jnb 1f
229 0: pushl $.kcopy_panic_msg
230 call panic
231 1: popl %ebp
232 #endif
233 lea _kcopy_copyerr, %eax /* lofault value */
234 movl %gs:CPU_THREAD, %edx
235
236 do_copy_fault:
237 pushl %ebp
238 movl %esp, %ebp /* setup stack frame */
239 pushl %esi
240 pushl %edi /* save registers */
241
242 movl T_LOFAULT(%edx), %edi
243 pushl %edi /* save the current lofault */
244 movl %eax, T_LOFAULT(%edx) /* new lofault */
245
246 movl ARG_COUNT(%ebp), %ecx
247 movl ARG_FROM(%ebp), %esi
248 movl ARG_TO(%ebp), %edi
249 shrl $2, %ecx /* word count */
250 rep
251 smovl
252 movl ARG_COUNT(%ebp), %ecx
253 andl $3, %ecx /* bytes left over */
254 rep
255 smovb
256 xorl %eax, %eax
257
258 /*
259 * A fault during do_copy_fault is indicated through an errno value
260 * in %eax and we iret from the trap handler to here.
261 */
262 _kcopy_copyerr:
263 popl %ecx
264 popl %edi
265 movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */
266 popl %esi
267 popl %ebp
268 ret
269 SET_SIZE(kcopy)
270
271 #undef ARG_FROM
272 #undef ARG_TO
273 #undef ARG_COUNT
274
275 #endif /* __i386 */
276 #endif /* __lint */
277
278 #if defined(__lint)
279
280 /*
281 * Copy a block of storage. Similar to kcopy but uses non-temporal
282 * instructions.
283 */
284
285 /* ARGSUSED */
286 int
287 kcopy_nta(const void *from, void *to, size_t count, int copy_cached)
288 { return (0); }
289
290 #else /* __lint */
291
292 #if defined(__amd64)
293
294 #define COPY_LOOP_INIT(src, dst, cnt) \
295 addq cnt, src; \
296 addq cnt, dst; \
297 shrq $3, cnt; \
298 neg cnt
299
300 /* Copy 16 bytes per loop. Uses %rax and %r8 */
301 #define COPY_LOOP_BODY(src, dst, cnt) \
302 prefetchnta 0x100(src, cnt, 8); \
303 movq (src, cnt, 8), %rax; \
304 movq 0x8(src, cnt, 8), %r8; \
305 movnti %rax, (dst, cnt, 8); \
306 movnti %r8, 0x8(dst, cnt, 8); \
307 addq $2, cnt
308
309 ENTRY(kcopy_nta)
310 pushq %rbp
311 movq %rsp, %rbp
312 #ifdef DEBUG
313 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
350 movq %rcx, T_LOFAULT(%r9) /* new lofault */
351
352 /*
353 * COPY_LOOP_BODY uses %rax and %r8
354 */
355 COPY_LOOP_INIT(%rdi, %rsi, %rdx)
356 2: COPY_LOOP_BODY(%rdi, %rsi, %rdx)
357 jnz 2b
358
359 mfence
360 xorl %eax, %eax /* return 0 (success) */
361 SMAP_ENABLE_INSTR(1)
362
363 _kcopy_nta_copyerr:
364 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
365 leave
366 ret
367 SET_SIZE(do_copy_fault_nta)
368 SET_SIZE(kcopy_nta)
369
370 #elif defined(__i386)
371
372 #define ARG_FROM 8
373 #define ARG_TO 12
374 #define ARG_COUNT 16
375
376 #define COPY_LOOP_INIT(src, dst, cnt) \
377 addl cnt, src; \
378 addl cnt, dst; \
379 shrl $3, cnt; \
380 neg cnt
381
382 #define COPY_LOOP_BODY(src, dst, cnt) \
383 prefetchnta 0x100(src, cnt, 8); \
384 movl (src, cnt, 8), %esi; \
385 movnti %esi, (dst, cnt, 8); \
386 movl 0x4(src, cnt, 8), %esi; \
387 movnti %esi, 0x4(dst, cnt, 8); \
388 movl 0x8(src, cnt, 8), %esi; \
389 movnti %esi, 0x8(dst, cnt, 8); \
390 movl 0xc(src, cnt, 8), %esi; \
391 movnti %esi, 0xc(dst, cnt, 8); \
392 addl $2, cnt
393
394 /*
395 * kcopy_nta is not implemented for 32-bit as no performance
396 * improvement was shown. We simply jump directly to kcopy
397 * and discard the 4 arguments.
398 */
399 ENTRY(kcopy_nta)
400 jmp kcopy
401
402 lea _kcopy_nta_copyerr, %eax /* lofault value */
403 ALTENTRY(do_copy_fault_nta)
404 pushl %ebp
405 movl %esp, %ebp /* setup stack frame */
406 pushl %esi
407 pushl %edi
408
409 movl %gs:CPU_THREAD, %edx
410 movl T_LOFAULT(%edx), %edi
411 pushl %edi /* save the current lofault */
412 movl %eax, T_LOFAULT(%edx) /* new lofault */
413
414 /* COPY_LOOP_BODY needs to use %esi */
415 movl ARG_COUNT(%ebp), %ecx
416 movl ARG_FROM(%ebp), %edi
417 movl ARG_TO(%ebp), %eax
418 COPY_LOOP_INIT(%edi, %eax, %ecx)
419 1: COPY_LOOP_BODY(%edi, %eax, %ecx)
420 jnz 1b
421 mfence
422
423 xorl %eax, %eax
424 _kcopy_nta_copyerr:
425 popl %ecx
426 popl %edi
427 movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */
428 popl %esi
429 leave
430 ret
431 SET_SIZE(do_copy_fault_nta)
432 SET_SIZE(kcopy_nta)
433
434 #undef ARG_FROM
435 #undef ARG_TO
436 #undef ARG_COUNT
437
438 #endif /* __i386 */
439 #endif /* __lint */
440
441 #if defined(__lint)
442
443 /* ARGSUSED */
444 void
445 bcopy(const void *from, void *to, size_t count)
446 {}
447
448 #else /* __lint */
449
450 #if defined(__amd64)
451
452 ENTRY(bcopy)
453 #ifdef DEBUG
454 orq %rdx, %rdx /* %rdx = count */
455 jz 1f
456 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
457 jb 0f
458 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
459 jnb 1f
460 0: leaq .bcopy_panic_msg(%rip), %rdi
461 jmp call_panic /* setup stack and call panic */
462 1:
463 #endif
464 /*
465 * bcopy_altentry() is called from kcopy, i.e., do_copy_fault.
466 * kcopy assumes that bcopy doesn't touch %r9 and %r11. If bcopy
467 * uses these registers in future they must be saved and restored.
468 */
469 ALTENTRY(bcopy_altentry)
470 do_copy:
471 #define L(s) .bcopy/**/s
960
961 #ifdef DEBUG
962 /*
963 * Setup frame on the run-time stack. The end of the input argument
964 * area must be aligned on a 16 byte boundary. The stack pointer %rsp,
965 * always points to the end of the latest allocated stack frame.
966 * panic(const char *format, ...) is a varargs function. When a
967 * function taking variable arguments is called, %rax must be set
968 * to eight times the number of floating point parameters passed
969 * to the function in SSE registers.
970 */
971 call_panic:
972 pushq %rbp /* align stack properly */
973 movq %rsp, %rbp
974 xorl %eax, %eax /* no variable arguments */
975 call panic /* %rdi = format string */
976 #endif
977 SET_SIZE(bcopy_altentry)
978 SET_SIZE(bcopy)
979
980 #elif defined(__i386)
981
982 #define ARG_FROM 4
983 #define ARG_TO 8
984 #define ARG_COUNT 12
985
986 ENTRY(bcopy)
987 #ifdef DEBUG
988 movl ARG_COUNT(%esp), %eax
989 orl %eax, %eax
990 jz 1f
991 movl postbootkernelbase, %eax
992 cmpl %eax, ARG_FROM(%esp)
993 jb 0f
994 cmpl %eax, ARG_TO(%esp)
995 jnb 1f
996 0: pushl %ebp
997 movl %esp, %ebp
998 pushl $.bcopy_panic_msg
999 call panic
1000 1:
1001 #endif
1002 do_copy:
1003 movl %esi, %eax /* save registers */
1004 movl %edi, %edx
1005 movl ARG_COUNT(%esp), %ecx
1006 movl ARG_FROM(%esp), %esi
1007 movl ARG_TO(%esp), %edi
1008
1009 shrl $2, %ecx /* word count */
1010 rep
1011 smovl
1012 movl ARG_COUNT(%esp), %ecx
1013 andl $3, %ecx /* bytes left over */
1014 rep
1015 smovb
1016 movl %eax, %esi /* restore registers */
1017 movl %edx, %edi
1018 ret
1019 SET_SIZE(bcopy)
1020
1021 #undef ARG_COUNT
1022 #undef ARG_FROM
1023 #undef ARG_TO
1024
1025 #endif /* __i386 */
1026 #endif /* __lint */
1027
1028
1029 /*
1030 * Zero a block of storage, returning an error code if we
1031 * take a kernel pagefault which cannot be resolved.
1032 * Returns errno value on pagefault error, 0 if all ok
1033 */
1034
1035 #if defined(__lint)
1036
1037 /* ARGSUSED */
1038 int
1039 kzero(void *addr, size_t count)
1040 { return (0); }
1041
1042 #else /* __lint */
1043
1044 #if defined(__amd64)
1045
1046 ENTRY(kzero)
1047 #ifdef DEBUG
1048 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */
1049 jnb 0f
1050 leaq .kzero_panic_msg(%rip), %rdi
1051 jmp call_panic /* setup stack and call panic */
1052 0:
1053 #endif
1054 /*
1055 * pass lofault value as 3rd argument for fault return
1056 */
1057 leaq _kzeroerr(%rip), %rdx
1058
1059 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
1060 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
1061 movq %rdx, T_LOFAULT(%r9) /* new lofault */
1062 call bzero_altentry
1063 xorl %eax, %eax
1064 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
1065 ret
1066 /*
1067 * A fault during bzero is indicated through an errno value
1068 * in %rax when we iretq to here.
1069 */
1070 _kzeroerr:
1071 addq $8, %rsp /* pop bzero_altentry call ret addr */
1072 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
1073 ret
1074 SET_SIZE(kzero)
1075
1076 #elif defined(__i386)
1077
1078 #define ARG_ADDR 8
1079 #define ARG_COUNT 12
1080
1081 ENTRY(kzero)
1082 #ifdef DEBUG
1083 pushl %ebp
1084 movl %esp, %ebp
1085 movl postbootkernelbase, %eax
1086 cmpl %eax, ARG_ADDR(%ebp)
1087 jnb 0f
1088 pushl $.kzero_panic_msg
1089 call panic
1090 0: popl %ebp
1091 #endif
1092 lea _kzeroerr, %eax /* kzeroerr is lofault value */
1093
1094 pushl %ebp /* save stack base */
1095 movl %esp, %ebp /* set new stack base */
1096 pushl %edi /* save %edi */
1097
1098 mov %gs:CPU_THREAD, %edx
1099 movl T_LOFAULT(%edx), %edi
1100 pushl %edi /* save the current lofault */
1101 movl %eax, T_LOFAULT(%edx) /* new lofault */
1102
1103 movl ARG_COUNT(%ebp), %ecx /* get size in bytes */
1104 movl ARG_ADDR(%ebp), %edi /* %edi <- address of bytes to clear */
1105 shrl $2, %ecx /* Count of double words to zero */
1106 xorl %eax, %eax /* sstol val */
1107 rep
1108 sstol /* %ecx contains words to clear (%eax=0) */
1109
1110 movl ARG_COUNT(%ebp), %ecx /* get size in bytes */
1111 andl $3, %ecx /* do mod 4 */
1112 rep
1113 sstob /* %ecx contains residual bytes to clear */
1114
1115 /*
1116 * A fault during kzero is indicated through an errno value
1117 * in %eax when we iret to here.
1118 */
1119 _kzeroerr:
1120 popl %edi
1121 movl %edi, T_LOFAULT(%edx) /* restore the original lofault */
1122 popl %edi
1123 popl %ebp
1124 ret
1125 SET_SIZE(kzero)
1126
1127 #undef ARG_ADDR
1128 #undef ARG_COUNT
1129
1130 #endif /* __i386 */
1131 #endif /* __lint */
1132
1133 /*
1134 * Zero a block of storage.
1135 */
1136
1137 #if defined(__lint)
1138
1139 /* ARGSUSED */
1140 void
1141 bzero(void *addr, size_t count)
1142 {}
1143
1144 #else /* __lint */
1145
1146 #if defined(__amd64)
1147
1148 ENTRY(bzero)
1149 #ifdef DEBUG
1150 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */
1151 jnb 0f
1152 leaq .bzero_panic_msg(%rip), %rdi
1153 jmp call_panic /* setup stack and call panic */
1154 0:
1155 #endif
1156 ALTENTRY(bzero_altentry)
1157 do_zero:
1158 #define L(s) .bzero/**/s
1159 xorl %eax, %eax
1160
1161 cmpq $0x50, %rsi /* 80 */
1162 jae L(ck_align)
1163
1164 /*
1165 * Performance data shows many caller's are zeroing small buffers. So
1166 * for best perf for these sizes unrolled code is used. Store zeros
1167 * without worrying about alignment.
1442 movslq (%r10,%rsi,4), %rcx
1443 leaq (%rcx,%r10,1), %r10
1444 INDIRECT_JMP_REG(r10)
1445
1446 /*
1447 * Use rep sstoq. Clear any remainder via unrolled code
1448 */
1449 .p2align 4
1450 L(use_rep):
1451 movq %rsi, %rcx /* get size in bytes */
1452 shrq $3, %rcx /* count of 8-byte words to zero */
1453 rep
1454 sstoq /* %rcx = words to clear (%rax=0) */
1455 andq $7, %rsi /* remaining bytes */
1456 jnz 9b
1457 ret
1458 #undef L
1459 SET_SIZE(bzero_altentry)
1460 SET_SIZE(bzero)
1461
1462 #elif defined(__i386)
1463
1464 #define ARG_ADDR 4
1465 #define ARG_COUNT 8
1466
1467 ENTRY(bzero)
1468 #ifdef DEBUG
1469 movl postbootkernelbase, %eax
1470 cmpl %eax, ARG_ADDR(%esp)
1471 jnb 0f
1472 pushl %ebp
1473 movl %esp, %ebp
1474 pushl $.bzero_panic_msg
1475 call panic
1476 0:
1477 #endif
1478 do_zero:
1479 movl %edi, %edx
1480 movl ARG_COUNT(%esp), %ecx
1481 movl ARG_ADDR(%esp), %edi
1482 shrl $2, %ecx
1483 xorl %eax, %eax
1484 rep
1485 sstol
1486 movl ARG_COUNT(%esp), %ecx
1487 andl $3, %ecx
1488 rep
1489 sstob
1490 movl %edx, %edi
1491 ret
1492 SET_SIZE(bzero)
1493
1494 #undef ARG_ADDR
1495 #undef ARG_COUNT
1496
1497 #endif /* __i386 */
1498 #endif /* __lint */
1499
1500 /*
1501 * Transfer data to and from user space -
1502 * Note that these routines can cause faults
1503 * It is assumed that the kernel has nothing at
1504 * less than KERNELBASE in the virtual address space.
1505 *
1506 * Note that copyin(9F) and copyout(9F) are part of the
1507 * DDI/DKI which specifies that they return '-1' on "errors."
1508 *
1509 * Sigh.
1510 *
1511 * So there's two extremely similar routines - xcopyin_nta() and
1512 * xcopyout_nta() which return the errno that we've faithfully computed.
1513 * This allows other callers (e.g. uiomove(9F)) to work correctly.
1514 * Given that these are used pretty heavily, we expand the calling
1515 * sequences inline for all flavours (rather than making wrappers).
1516 */
1517
1518 /*
1519 * Copy user data to kernel space.
1520 */
1521
1522 #if defined(__lint)
1523
1524 /* ARGSUSED */
1525 int
1526 copyin(const void *uaddr, void *kaddr, size_t count)
1527 { return (0); }
1528
1529 #else /* lint */
1530
1531 #if defined(__amd64)
1532
1533 ENTRY(copyin)
1534 pushq %rbp
1535 movq %rsp, %rbp
1536 subq $24, %rsp
1537
1538 /*
1539 * save args in case we trap and need to rerun as a copyop
1540 */
1541 movq %rdi, (%rsp)
1542 movq %rsi, 0x8(%rsp)
1543 movq %rdx, 0x10(%rsp)
1544
1545 movq kernelbase(%rip), %rax
1546 #ifdef DEBUG
1547 cmpq %rax, %rsi /* %rsi = kaddr */
1548 jnb 1f
1549 leaq .copyin_panic_msg(%rip), %rdi
1550 xorl %eax, %eax
1551 call panic
1552 1:
1568 addq $8, %rsp /* pop bcopy_altentry call ret addr */
1569 3:
1570 movq T_COPYOPS(%r9), %rax
1571 cmpq $0, %rax
1572 jz 2f
1573 /*
1574 * reload args for the copyop
1575 */
1576 movq (%rsp), %rdi
1577 movq 0x8(%rsp), %rsi
1578 movq 0x10(%rsp), %rdx
1579 leave
1580 movq CP_COPYIN(%rax), %rax
1581 INDIRECT_JMP_REG(rax)
1582
1583 2: movl $-1, %eax
1584 leave
1585 ret
1586 SET_SIZE(copyin)
1587
1588 #elif defined(__i386)
1589
1590 #define ARG_UADDR 4
1591 #define ARG_KADDR 8
1592
1593 ENTRY(copyin)
1594 movl kernelbase, %ecx
1595 #ifdef DEBUG
1596 cmpl %ecx, ARG_KADDR(%esp)
1597 jnb 1f
1598 pushl %ebp
1599 movl %esp, %ebp
1600 pushl $.copyin_panic_msg
1601 call panic
1602 1:
1603 #endif
1604 lea _copyin_err, %eax
1605
1606 movl %gs:CPU_THREAD, %edx
1607 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
1608 jb do_copy_fault
1609 jmp 3f
1610
1611 _copyin_err:
1612 popl %ecx
1613 popl %edi
1614 movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
1615 popl %esi
1616 popl %ebp
1617 3:
1618 movl T_COPYOPS(%edx), %eax
1619 cmpl $0, %eax
1620 jz 2f
1621 jmp *CP_COPYIN(%eax)
1622
1623 2: movl $-1, %eax
1624 ret
1625 SET_SIZE(copyin)
1626
1627 #undef ARG_UADDR
1628 #undef ARG_KADDR
1629
1630 #endif /* __i386 */
1631 #endif /* __lint */
1632
1633 #if defined(__lint)
1634
1635 /* ARGSUSED */
1636 int
1637 xcopyin_nta(const void *uaddr, void *kaddr, size_t count, int copy_cached)
1638 { return (0); }
1639
1640 #else /* __lint */
1641
1642 #if defined(__amd64)
1643
1644 ENTRY(xcopyin_nta)
1645 pushq %rbp
1646 movq %rsp, %rbp
1647 subq $24, %rsp
1648
1649 /*
1650 * save args in case we trap and need to rerun as a copyop
1651 * %rcx is consumed in this routine so we don't need to save
1652 * it.
1653 */
1654 movq %rdi, (%rsp)
1655 movq %rsi, 0x8(%rsp)
1656 movq %rdx, 0x10(%rsp)
1657
1658 movq kernelbase(%rip), %rax
1659 #ifdef DEBUG
1660 cmpq %rax, %rsi /* %rsi = kaddr */
1661 jnb 1f
1662 leaq .xcopyin_panic_msg(%rip), %rdi
1663 xorl %eax, %eax
1713 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
1714 3:
1715 movq T_COPYOPS(%r9), %r8
1716 cmpq $0, %r8
1717 jz 2f
1718
1719 /*
1720 * reload args for the copyop
1721 */
1722 movq (%rsp), %rdi
1723 movq 0x8(%rsp), %rsi
1724 movq 0x10(%rsp), %rdx
1725 leave
1726 movq CP_XCOPYIN(%r8), %r8
1727 INDIRECT_JMP_REG(r8)
1728
1729 2: leave
1730 ret
1731 SET_SIZE(xcopyin_nta)
1732
1733 #elif defined(__i386)
1734
1735 #define ARG_UADDR 4
1736 #define ARG_KADDR 8
1737 #define ARG_COUNT 12
1738 #define ARG_CACHED 16
1739
1740 .globl use_sse_copy
1741
1742 ENTRY(xcopyin_nta)
1743 movl kernelbase, %ecx
1744 lea _xcopyin_err, %eax
1745 movl %gs:CPU_THREAD, %edx
1746 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
1747 jae 4f
1748
1749 cmpl $0, use_sse_copy /* no sse support */
1750 jz do_copy_fault
1751
1752 cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */
1753 jnz do_copy_fault
1754
1755 /*
1756 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
1757 */
1758 cmpl $XCOPY_MIN_SIZE, ARG_COUNT(%esp)
1759 jb do_copy_fault
1760
1761 /*
1762 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
1763 * count is COUNT_ALIGN_SIZE aligned.
1764 */
1765 movl ARG_UADDR(%esp), %ecx
1766 orl ARG_KADDR(%esp), %ecx
1767 andl $NTA_ALIGN_MASK, %ecx
1768 orl ARG_COUNT(%esp), %ecx
1769 andl $COUNT_ALIGN_MASK, %ecx
1770 jnz do_copy_fault
1771
1772 jmp do_copy_fault_nta /* use regular access */
1773
1774 4:
1775 movl $EFAULT, %eax
1776 jmp 3f
1777
1778 /*
1779 * A fault during do_copy_fault or do_copy_fault_nta is
1780 * indicated through an errno value in %eax and we iret from the
1781 * trap handler to here.
1782 */
1783 _xcopyin_err:
1784 popl %ecx
1785 popl %edi
1786 movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
1787 popl %esi
1788 popl %ebp
1789 3:
1790 cmpl $0, T_COPYOPS(%edx)
1791 jz 2f
1792 movl T_COPYOPS(%edx), %eax
1793 jmp *CP_XCOPYIN(%eax)
1794
1795 2: rep; ret /* use 2 byte return instruction when branch target */
1796 /* AMD Software Optimization Guide - Section 6.2 */
1797 SET_SIZE(xcopyin_nta)
1798
1799 #undef ARG_UADDR
1800 #undef ARG_KADDR
1801 #undef ARG_COUNT
1802 #undef ARG_CACHED
1803
1804 #endif /* __i386 */
1805 #endif /* __lint */
1806
1807 /*
1808 * Copy kernel data to user space.
1809 */
1810
1811 #if defined(__lint)
1812
1813 /* ARGSUSED */
1814 int
1815 copyout(const void *kaddr, void *uaddr, size_t count)
1816 { return (0); }
1817
1818 #else /* __lint */
1819
1820 #if defined(__amd64)
1821
1822 ENTRY(copyout)
1823 pushq %rbp
1824 movq %rsp, %rbp
1825 subq $24, %rsp
1826
1827 /*
1828 * save args in case we trap and need to rerun as a copyop
1829 */
1830 movq %rdi, (%rsp)
1831 movq %rsi, 0x8(%rsp)
1832 movq %rdx, 0x10(%rsp)
1833
1834 movq kernelbase(%rip), %rax
1835 #ifdef DEBUG
1836 cmpq %rax, %rdi /* %rdi = kaddr */
1837 jnb 1f
1838 leaq .copyout_panic_msg(%rip), %rdi
1839 xorl %eax, %eax
1840 call panic
1841 1:
1858 3:
1859 movq T_COPYOPS(%r9), %rax
1860 cmpq $0, %rax
1861 jz 2f
1862
1863 /*
1864 * reload args for the copyop
1865 */
1866 movq (%rsp), %rdi
1867 movq 0x8(%rsp), %rsi
1868 movq 0x10(%rsp), %rdx
1869 leave
1870 movq CP_COPYOUT(%rax), %rax
1871 INDIRECT_JMP_REG(rax)
1872
1873 2: movl $-1, %eax
1874 leave
1875 ret
1876 SET_SIZE(copyout)
1877
1878 #elif defined(__i386)
1879
1880 #define ARG_KADDR 4
1881 #define ARG_UADDR 8
1882
1883 ENTRY(copyout)
1884 movl kernelbase, %ecx
1885 #ifdef DEBUG
1886 cmpl %ecx, ARG_KADDR(%esp)
1887 jnb 1f
1888 pushl %ebp
1889 movl %esp, %ebp
1890 pushl $.copyout_panic_msg
1891 call panic
1892 1:
1893 #endif
1894 lea _copyout_err, %eax
1895 movl %gs:CPU_THREAD, %edx
1896 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
1897 jb do_copy_fault
1898 jmp 3f
1899
1900 _copyout_err:
1901 popl %ecx
1902 popl %edi
1903 movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
1904 popl %esi
1905 popl %ebp
1906 3:
1907 movl T_COPYOPS(%edx), %eax
1908 cmpl $0, %eax
1909 jz 2f
1910 jmp *CP_COPYOUT(%eax)
1911
1912 2: movl $-1, %eax
1913 ret
1914 SET_SIZE(copyout)
1915
1916 #undef ARG_UADDR
1917 #undef ARG_KADDR
1918
1919 #endif /* __i386 */
1920 #endif /* __lint */
1921
1922 #if defined(__lint)
1923
1924 /* ARGSUSED */
1925 int
1926 xcopyout_nta(const void *kaddr, void *uaddr, size_t count, int copy_cached)
1927 { return (0); }
1928
1929 #else /* __lint */
1930
1931 #if defined(__amd64)
1932
1933 ENTRY(xcopyout_nta)
1934 pushq %rbp
1935 movq %rsp, %rbp
1936 subq $24, %rsp
1937
1938 /*
1939 * save args in case we trap and need to rerun as a copyop
1940 */
1941 movq %rdi, (%rsp)
1942 movq %rsi, 0x8(%rsp)
1943 movq %rdx, 0x10(%rsp)
1944
1945 movq kernelbase(%rip), %rax
1946 #ifdef DEBUG
1947 cmpq %rax, %rdi /* %rdi = kaddr */
1948 jnb 1f
1949 leaq .xcopyout_panic_msg(%rip), %rdi
1950 xorl %eax, %eax
1951 call panic
1952 1:
2003 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
2004 3:
2005 movq T_COPYOPS(%r9), %r8
2006 cmpq $0, %r8
2007 jz 2f
2008
2009 /*
2010 * reload args for the copyop
2011 */
2012 movq (%rsp), %rdi
2013 movq 0x8(%rsp), %rsi
2014 movq 0x10(%rsp), %rdx
2015 leave
2016 movq CP_XCOPYOUT(%r8), %r8
2017 INDIRECT_JMP_REG(r8)
2018
2019 2: leave
2020 ret
2021 SET_SIZE(xcopyout_nta)
2022
2023 #elif defined(__i386)
2024
2025 #define ARG_KADDR 4
2026 #define ARG_UADDR 8
2027 #define ARG_COUNT 12
2028 #define ARG_CACHED 16
2029
2030 ENTRY(xcopyout_nta)
2031 movl kernelbase, %ecx
2032 lea _xcopyout_err, %eax
2033 movl %gs:CPU_THREAD, %edx
2034 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
2035 jae 4f
2036
2037 cmpl $0, use_sse_copy /* no sse support */
2038 jz do_copy_fault
2039
2040 cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */
2041 jnz do_copy_fault
2042
2043 /*
2044 * Make sure cnt is >= XCOPY_MIN_SIZE bytes
2045 */
2046 cmpl $XCOPY_MIN_SIZE, %edx
2047 jb do_copy_fault
2048
2049 /*
2050 * Make sure src and dst are NTA_ALIGN_SIZE aligned,
2051 * count is COUNT_ALIGN_SIZE aligned.
2052 */
2053 movl ARG_UADDR(%esp), %ecx
2054 orl ARG_KADDR(%esp), %ecx
2055 andl $NTA_ALIGN_MASK, %ecx
2056 orl ARG_COUNT(%esp), %ecx
2057 andl $COUNT_ALIGN_MASK, %ecx
2058 jnz do_copy_fault
2059 jmp do_copy_fault_nta
2060
2061 4:
2062 movl $EFAULT, %eax
2063 jmp 3f
2064
2065 /*
2066 * A fault during do_copy_fault or do_copy_fault_nta is
2067 * indicated through an errno value in %eax and we iret from the
2068 * trap handler to here.
2069 */
2070 _xcopyout_err:
2071 / restore the original lofault
2072 popl %ecx
2073 popl %edi
2074 movl %ecx, T_LOFAULT(%edx) / original lofault
2075 popl %esi
2076 popl %ebp
2077 3:
2078 cmpl $0, T_COPYOPS(%edx)
2079 jz 2f
2080 movl T_COPYOPS(%edx), %eax
2081 jmp *CP_XCOPYOUT(%eax)
2082
2083 2: rep; ret /* use 2 byte return instruction when branch target */
2084 /* AMD Software Optimization Guide - Section 6.2 */
2085 SET_SIZE(xcopyout_nta)
2086
2087 #undef ARG_UADDR
2088 #undef ARG_KADDR
2089 #undef ARG_COUNT
2090 #undef ARG_CACHED
2091
2092 #endif /* __i386 */
2093 #endif /* __lint */
2094
2095 /*
2096 * Copy a null terminated string from one point to another in
2097 * the kernel address space.
2098 */
2099
2100 #if defined(__lint)
2101
2102 /* ARGSUSED */
2103 int
2104 copystr(const char *from, char *to, size_t maxlength, size_t *lencopied)
2105 { return (0); }
2106
2107 #else /* __lint */
2108
2109 #if defined(__amd64)
2110
2111 ENTRY(copystr)
2112 pushq %rbp
2113 movq %rsp, %rbp
2114 #ifdef DEBUG
2115 movq kernelbase(%rip), %rax
2116 cmpq %rax, %rdi /* %rdi = from */
2117 jb 0f
2118 cmpq %rax, %rsi /* %rsi = to */
2119 jnb 1f
2120 0: leaq .copystr_panic_msg(%rip), %rdi
2121 xorl %eax, %eax
2122 call panic
2123 1:
2124 #endif
2125 movq %gs:CPU_THREAD, %r9
2126 movq T_LOFAULT(%r9), %r8 /* pass current lofault value as */
2127 /* 5th argument to do_copystr */
2128 xorl %r10d,%r10d /* pass smap restore need in %r10d */
2129 /* as a non-ABI 6th arg */
2130 do_copystr:
2155 copystr_null:
2156 xorl %eax, %eax /* no error */
2157
2158 copystr_out:
2159 cmpq $0, %rcx /* want length? */
2160 je copystr_smap /* no */
2161 subq %r8, %rdx /* compute length and store it */
2162 movq %rdx, (%rcx)
2163
2164 copystr_smap:
2165 cmpl $0, %r10d
2166 jz copystr_done
2167 SMAP_ENABLE_INSTR(7)
2168
2169 copystr_done:
2170 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
2171 leave
2172 ret
2173 SET_SIZE(copystr)
2174
2175 #elif defined(__i386)
2176
2177 #define ARG_FROM 8
2178 #define ARG_TO 12
2179 #define ARG_MAXLEN 16
2180 #define ARG_LENCOPIED 20
2181
2182 ENTRY(copystr)
2183 #ifdef DEBUG
2184 pushl %ebp
2185 movl %esp, %ebp
2186 movl kernelbase, %eax
2187 cmpl %eax, ARG_FROM(%esp)
2188 jb 0f
2189 cmpl %eax, ARG_TO(%esp)
2190 jnb 1f
2191 0: pushl $.copystr_panic_msg
2192 call panic
2193 1: popl %ebp
2194 #endif
2195 /* get the current lofault address */
2196 movl %gs:CPU_THREAD, %eax
2197 movl T_LOFAULT(%eax), %eax
2198 do_copystr:
2199 pushl %ebp /* setup stack frame */
2200 movl %esp, %ebp
2201 pushl %ebx /* save registers */
2202 pushl %edi
2203
2204 movl %gs:CPU_THREAD, %ebx
2205 movl T_LOFAULT(%ebx), %edi
2206 pushl %edi /* save the current lofault */
2207 movl %eax, T_LOFAULT(%ebx) /* new lofault */
2208
2209 movl ARG_MAXLEN(%ebp), %ecx
2210 cmpl $0, %ecx
2211 je copystr_enametoolong /* maxlength == 0 */
2212
2213 movl ARG_FROM(%ebp), %ebx /* source address */
2214 movl ARG_TO(%ebp), %edx /* destination address */
2215
2216 copystr_loop:
2217 decl %ecx
2218 movb (%ebx), %al
2219 incl %ebx
2220 movb %al, (%edx)
2221 incl %edx
2222 cmpb $0, %al
2223 je copystr_null /* null char */
2224 cmpl $0, %ecx
2225 jne copystr_loop
2226
2227 copystr_enametoolong:
2228 movl $ENAMETOOLONG, %eax
2229 jmp copystr_out
2230
2231 copystr_null:
2232 xorl %eax, %eax /* no error */
2233
2234 copystr_out:
2235 cmpl $0, ARG_LENCOPIED(%ebp) /* want length? */
2236 je copystr_done /* no */
2237 movl ARG_MAXLEN(%ebp), %edx
2238 subl %ecx, %edx /* compute length and store it */
2239 movl ARG_LENCOPIED(%ebp), %ecx
2240 movl %edx, (%ecx)
2241
2242 copystr_done:
2243 popl %edi
2244 movl %gs:CPU_THREAD, %ebx
2245 movl %edi, T_LOFAULT(%ebx) /* restore the original lofault */
2246
2247 popl %edi
2248 popl %ebx
2249 popl %ebp
2250 ret
2251 SET_SIZE(copystr)
2252
2253 #undef ARG_FROM
2254 #undef ARG_TO
2255 #undef ARG_MAXLEN
2256 #undef ARG_LENCOPIED
2257
2258 #endif /* __i386 */
2259 #endif /* __lint */
2260
2261 /*
2262 * Copy a null terminated string from the user address space into
2263 * the kernel address space.
2264 */
2265
2266 #if defined(__lint)
2267
2268 /* ARGSUSED */
2269 int
2270 copyinstr(const char *uaddr, char *kaddr, size_t maxlength,
2271 size_t *lencopied)
2272 { return (0); }
2273
2274 #else /* __lint */
2275
2276 #if defined(__amd64)
2277
2278 ENTRY(copyinstr)
2279 pushq %rbp
2280 movq %rsp, %rbp
2281 subq $32, %rsp
2282
2283 /*
2284 * save args in case we trap and need to rerun as a copyop
2285 */
2286 movq %rdi, (%rsp)
2287 movq %rsi, 0x8(%rsp)
2288 movq %rdx, 0x10(%rsp)
2289 movq %rcx, 0x18(%rsp)
2290
2291 movq kernelbase(%rip), %rax
2292 #ifdef DEBUG
2293 cmpq %rax, %rsi /* %rsi = kaddr */
2294 jnb 1f
2295 leaq .copyinstr_panic_msg(%rip), %rdi
2296 xorl %eax, %eax
2297 call panic
2319 movq T_COPYOPS(%r9), %rax
2320 cmpq $0, %rax
2321 jz 2f
2322
2323 /*
2324 * reload args for the copyop
2325 */
2326 movq (%rsp), %rdi
2327 movq 0x8(%rsp), %rsi
2328 movq 0x10(%rsp), %rdx
2329 movq 0x18(%rsp), %rcx
2330 leave
2331 movq CP_COPYINSTR(%rax), %rax
2332 INDIRECT_JMP_REG(rax)
2333
2334 2: movl $EFAULT, %eax /* return EFAULT */
2335 leave
2336 ret
2337 SET_SIZE(copyinstr)
2338
2339 #elif defined(__i386)
2340
2341 #define ARG_UADDR 4
2342 #define ARG_KADDR 8
2343
2344 ENTRY(copyinstr)
2345 movl kernelbase, %ecx
2346 #ifdef DEBUG
2347 cmpl %ecx, ARG_KADDR(%esp)
2348 jnb 1f
2349 pushl %ebp
2350 movl %esp, %ebp
2351 pushl $.copyinstr_panic_msg
2352 call panic
2353 1:
2354 #endif
2355 lea _copyinstr_error, %eax
2356 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
2357 jb do_copystr
2358 movl %gs:CPU_THREAD, %edx
2359 jmp 3f
2360
2361 _copyinstr_error:
2362 popl %edi
2363 movl %gs:CPU_THREAD, %edx
2364 movl %edi, T_LOFAULT(%edx) /* original lofault */
2365
2366 popl %edi
2367 popl %ebx
2368 popl %ebp
2369 3:
2370 movl T_COPYOPS(%edx), %eax
2371 cmpl $0, %eax
2372 jz 2f
2373 jmp *CP_COPYINSTR(%eax)
2374
2375 2: movl $EFAULT, %eax /* return EFAULT */
2376 ret
2377 SET_SIZE(copyinstr)
2378
2379 #undef ARG_UADDR
2380 #undef ARG_KADDR
2381
2382 #endif /* __i386 */
2383 #endif /* __lint */
2384
2385 /*
2386 * Copy a null terminated string from the kernel
2387 * address space to the user address space.
2388 */
2389
2390 #if defined(__lint)
2391
2392 /* ARGSUSED */
2393 int
2394 copyoutstr(const char *kaddr, char *uaddr, size_t maxlength,
2395 size_t *lencopied)
2396 { return (0); }
2397
2398 #else /* __lint */
2399
2400 #if defined(__amd64)
2401
2402 ENTRY(copyoutstr)
2403 pushq %rbp
2404 movq %rsp, %rbp
2405 subq $32, %rsp
2406
2407 /*
2408 * save args in case we trap and need to rerun as a copyop
2409 */
2410 movq %rdi, (%rsp)
2411 movq %rsi, 0x8(%rsp)
2412 movq %rdx, 0x10(%rsp)
2413 movq %rcx, 0x18(%rsp)
2414
2415 movq kernelbase(%rip), %rax
2416 #ifdef DEBUG
2417 cmpq %rax, %rdi /* %rdi = kaddr */
2418 jnb 1f
2419 leaq .copyoutstr_panic_msg(%rip), %rdi
2420 jmp call_panic /* setup stack and call panic */
2421 1:
2442 movq T_COPYOPS(%r9), %rax
2443 cmpq $0, %rax
2444 jz 2f
2445
2446 /*
2447 * reload args for the copyop
2448 */
2449 movq (%rsp), %rdi
2450 movq 0x8(%rsp), %rsi
2451 movq 0x10(%rsp), %rdx
2452 movq 0x18(%rsp), %rcx
2453 leave
2454 movq CP_COPYOUTSTR(%rax), %rax
2455 INDIRECT_JMP_REG(rax)
2456
2457 2: movl $EFAULT, %eax /* return EFAULT */
2458 leave
2459 ret
2460 SET_SIZE(copyoutstr)
2461
2462 #elif defined(__i386)
2463
2464 #define ARG_KADDR 4
2465 #define ARG_UADDR 8
2466
2467 ENTRY(copyoutstr)
2468 movl kernelbase, %ecx
2469 #ifdef DEBUG
2470 cmpl %ecx, ARG_KADDR(%esp)
2471 jnb 1f
2472 pushl %ebp
2473 movl %esp, %ebp
2474 pushl $.copyoutstr_panic_msg
2475 call panic
2476 1:
2477 #endif
2478 lea _copyoutstr_error, %eax
2479 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
2480 jb do_copystr
2481 movl %gs:CPU_THREAD, %edx
2482 jmp 3f
2483
2484 _copyoutstr_error:
2485 popl %edi
2486 movl %gs:CPU_THREAD, %edx
2487 movl %edi, T_LOFAULT(%edx) /* restore the original lofault */
2488
2489 popl %edi
2490 popl %ebx
2491 popl %ebp
2492 3:
2493 movl T_COPYOPS(%edx), %eax
2494 cmpl $0, %eax
2495 jz 2f
2496 jmp *CP_COPYOUTSTR(%eax)
2497
2498 2: movl $EFAULT, %eax /* return EFAULT */
2499 ret
2500 SET_SIZE(copyoutstr)
2501
2502 #undef ARG_KADDR
2503 #undef ARG_UADDR
2504
2505 #endif /* __i386 */
2506 #endif /* __lint */
2507
2508 /*
2509 * Since all of the fuword() variants are so similar, we have a macro to spit
2510 * them out. This allows us to create DTrace-unobservable functions easily.
2511 */
2512
2513 #if defined(__lint)
2514
2515 #if defined(__amd64)
2516
2517 /* ARGSUSED */
2518 int
2519 fuword64(const void *addr, uint64_t *dst)
2520 { return (0); }
2521
2522 #endif
2523
2524 /* ARGSUSED */
2525 int
2526 fuword32(const void *addr, uint32_t *dst)
2527 { return (0); }
2528
2529 /* ARGSUSED */
2530 int
2531 fuword16(const void *addr, uint16_t *dst)
2532 { return (0); }
2533
2534 /* ARGSUSED */
2535 int
2536 fuword8(const void *addr, uint8_t *dst)
2537 { return (0); }
2538
2539 #else /* __lint */
2540
2541 #if defined(__amd64)
2542
2543 /*
2544 * Note that we don't save and reload the arguments here
2545 * because their values are not altered in the copy path.
2546 * Additionally, when successful, the smap_enable jmp will
2547 * actually return us to our original caller.
2548 */
2549
2550 #define FUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2) \
2551 ENTRY(NAME) \
2552 movq %gs:CPU_THREAD, %r9; \
2553 cmpq kernelbase(%rip), %rdi; \
2554 jae 1f; \
2555 leaq _flt_/**/NAME, %rdx; \
2556 movq %rdx, T_LOFAULT(%r9); \
2557 SMAP_DISABLE_INSTR(DISNUM) \
2558 INSTR (%rdi), REG; \
2559 movq $0, T_LOFAULT(%r9); \
2560 INSTR REG, (%rsi); \
2561 xorl %eax, %eax; \
2562 SMAP_ENABLE_INSTR(EN1) \
2563 ret; \
2564 _flt_/**/NAME: \
2565 SMAP_ENABLE_INSTR(EN2) \
2566 movq $0, T_LOFAULT(%r9); \
2567 1: \
2568 movq T_COPYOPS(%r9), %rax; \
2569 cmpq $0, %rax; \
2570 jz 2f; \
2571 movq COPYOP(%rax), %rax; \
2572 INDIRECT_JMP_REG(rax); \
2573 2: \
2574 movl $-1, %eax; \
2575 ret; \
2576 SET_SIZE(NAME)
2577
2578 FUWORD(fuword64, movq, %rax, CP_FUWORD64,8,10,11)
2579 FUWORD(fuword32, movl, %eax, CP_FUWORD32,9,12,13)
2580 FUWORD(fuword16, movw, %ax, CP_FUWORD16,10,14,15)
2581 FUWORD(fuword8, movb, %al, CP_FUWORD8,11,16,17)
2582
2583 #elif defined(__i386)
2584
2585 #define FUWORD(NAME, INSTR, REG, COPYOP) \
2586 ENTRY(NAME) \
2587 movl %gs:CPU_THREAD, %ecx; \
2588 movl kernelbase, %eax; \
2589 cmpl %eax, 4(%esp); \
2590 jae 1f; \
2591 lea _flt_/**/NAME, %edx; \
2592 movl %edx, T_LOFAULT(%ecx); \
2593 movl 4(%esp), %eax; \
2594 movl 8(%esp), %edx; \
2595 INSTR (%eax), REG; \
2596 movl $0, T_LOFAULT(%ecx); \
2597 INSTR REG, (%edx); \
2598 xorl %eax, %eax; \
2599 ret; \
2600 _flt_/**/NAME: \
2601 movl $0, T_LOFAULT(%ecx); \
2602 1: \
2603 movl T_COPYOPS(%ecx), %eax; \
2604 cmpl $0, %eax; \
2605 jz 2f; \
2606 jmp *COPYOP(%eax); \
2607 2: \
2608 movl $-1, %eax; \
2609 ret; \
2610 SET_SIZE(NAME)
2611
2612 FUWORD(fuword32, movl, %eax, CP_FUWORD32)
2613 FUWORD(fuword16, movw, %ax, CP_FUWORD16)
2614 FUWORD(fuword8, movb, %al, CP_FUWORD8)
2615
2616 #endif /* __i386 */
2617
2618 #undef FUWORD
2619
2620 #endif /* __lint */
2621
2622 /*
2623 * Set user word.
2624 */
2625
2626 #if defined(__lint)
2627
2628 #if defined(__amd64)
2629
2630 /* ARGSUSED */
2631 int
2632 suword64(void *addr, uint64_t value)
2633 { return (0); }
2634
2635 #endif
2636
2637 /* ARGSUSED */
2638 int
2639 suword32(void *addr, uint32_t value)
2640 { return (0); }
2641
2642 /* ARGSUSED */
2643 int
2644 suword16(void *addr, uint16_t value)
2645 { return (0); }
2646
2647 /* ARGSUSED */
2648 int
2649 suword8(void *addr, uint8_t value)
2650 { return (0); }
2651
2652 #else /* lint */
2653
2654 #if defined(__amd64)
2655
2656 /*
2657 * Note that we don't save and reload the arguments here
2658 * because their values are not altered in the copy path.
2659 */
2660
2661 #define SUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2) \
2662 ENTRY(NAME) \
2663 movq %gs:CPU_THREAD, %r9; \
2664 cmpq kernelbase(%rip), %rdi; \
2665 jae 1f; \
2666 leaq _flt_/**/NAME, %rdx; \
2667 SMAP_DISABLE_INSTR(DISNUM) \
2668 movq %rdx, T_LOFAULT(%r9); \
2669 INSTR REG, (%rdi); \
2670 movq $0, T_LOFAULT(%r9); \
2671 xorl %eax, %eax; \
2672 SMAP_ENABLE_INSTR(EN1) \
2673 ret; \
2674 _flt_/**/NAME: \
2675 SMAP_ENABLE_INSTR(EN2) \
2676 movq $0, T_LOFAULT(%r9); \
2677 1: \
2678 movq T_COPYOPS(%r9), %rax; \
2679 cmpq $0, %rax; \
2680 jz 3f; \
2681 movq COPYOP(%rax), %rax; \
2682 INDIRECT_JMP_REG(rax); \
2683 3: \
2684 movl $-1, %eax; \
2685 ret; \
2686 SET_SIZE(NAME)
2687
2688 SUWORD(suword64, movq, %rsi, CP_SUWORD64,12,18,19)
2689 SUWORD(suword32, movl, %esi, CP_SUWORD32,13,20,21)
2690 SUWORD(suword16, movw, %si, CP_SUWORD16,14,22,23)
2691 SUWORD(suword8, movb, %sil, CP_SUWORD8,15,24,25)
2692
2693 #elif defined(__i386)
2694
2695 #define SUWORD(NAME, INSTR, REG, COPYOP) \
2696 ENTRY(NAME) \
2697 movl %gs:CPU_THREAD, %ecx; \
2698 movl kernelbase, %eax; \
2699 cmpl %eax, 4(%esp); \
2700 jae 1f; \
2701 lea _flt_/**/NAME, %edx; \
2702 movl %edx, T_LOFAULT(%ecx); \
2703 movl 4(%esp), %eax; \
2704 movl 8(%esp), %edx; \
2705 INSTR REG, (%eax); \
2706 movl $0, T_LOFAULT(%ecx); \
2707 xorl %eax, %eax; \
2708 ret; \
2709 _flt_/**/NAME: \
2710 movl $0, T_LOFAULT(%ecx); \
2711 1: \
2712 movl T_COPYOPS(%ecx), %eax; \
2713 cmpl $0, %eax; \
2714 jz 3f; \
2715 movl COPYOP(%eax), %ecx; \
2716 jmp *%ecx; \
2717 3: \
2718 movl $-1, %eax; \
2719 ret; \
2720 SET_SIZE(NAME)
2721
2722 SUWORD(suword32, movl, %edx, CP_SUWORD32)
2723 SUWORD(suword16, movw, %dx, CP_SUWORD16)
2724 SUWORD(suword8, movb, %dl, CP_SUWORD8)
2725
2726 #endif /* __i386 */
2727
2728 #undef SUWORD
2729
2730 #endif /* __lint */
2731
2732 #if defined(__lint)
2733
2734 #if defined(__amd64)
2735
2736 /*ARGSUSED*/
2737 void
2738 fuword64_noerr(const void *addr, uint64_t *dst)
2739 {}
2740
2741 #endif
2742
2743 /*ARGSUSED*/
2744 void
2745 fuword32_noerr(const void *addr, uint32_t *dst)
2746 {}
2747
2748 /*ARGSUSED*/
2749 void
2750 fuword8_noerr(const void *addr, uint8_t *dst)
2751 {}
2752
2753 /*ARGSUSED*/
2754 void
2755 fuword16_noerr(const void *addr, uint16_t *dst)
2756 {}
2757
2758 #else /* __lint */
2759
2760 #if defined(__amd64)
2761
2762 #define FUWORD_NOERR(NAME, INSTR, REG) \
2763 ENTRY(NAME) \
2764 cmpq kernelbase(%rip), %rdi; \
2765 cmovnbq kernelbase(%rip), %rdi; \
2766 INSTR (%rdi), REG; \
2767 INSTR REG, (%rsi); \
2768 ret; \
2769 SET_SIZE(NAME)
2770
2771 FUWORD_NOERR(fuword64_noerr, movq, %rax)
2772 FUWORD_NOERR(fuword32_noerr, movl, %eax)
2773 FUWORD_NOERR(fuword16_noerr, movw, %ax)
2774 FUWORD_NOERR(fuword8_noerr, movb, %al)
2775
2776 #elif defined(__i386)
2777
2778 #define FUWORD_NOERR(NAME, INSTR, REG) \
2779 ENTRY(NAME) \
2780 movl 4(%esp), %eax; \
2781 cmpl kernelbase, %eax; \
2782 jb 1f; \
2783 movl kernelbase, %eax; \
2784 1: movl 8(%esp), %edx; \
2785 INSTR (%eax), REG; \
2786 INSTR REG, (%edx); \
2787 ret; \
2788 SET_SIZE(NAME)
2789
2790 FUWORD_NOERR(fuword32_noerr, movl, %ecx)
2791 FUWORD_NOERR(fuword16_noerr, movw, %cx)
2792 FUWORD_NOERR(fuword8_noerr, movb, %cl)
2793
2794 #endif /* __i386 */
2795
2796 #undef FUWORD_NOERR
2797
2798 #endif /* __lint */
2799
2800 #if defined(__lint)
2801
2802 #if defined(__amd64)
2803
2804 /*ARGSUSED*/
2805 void
2806 suword64_noerr(void *addr, uint64_t value)
2807 {}
2808
2809 #endif
2810
2811 /*ARGSUSED*/
2812 void
2813 suword32_noerr(void *addr, uint32_t value)
2814 {}
2815
2816 /*ARGSUSED*/
2817 void
2818 suword16_noerr(void *addr, uint16_t value)
2819 {}
2820
2821 /*ARGSUSED*/
2822 void
2823 suword8_noerr(void *addr, uint8_t value)
2824 {}
2825
2826 #else /* lint */
2827
2828 #if defined(__amd64)
2829
2830 #define SUWORD_NOERR(NAME, INSTR, REG) \
2831 ENTRY(NAME) \
2832 cmpq kernelbase(%rip), %rdi; \
2833 cmovnbq kernelbase(%rip), %rdi; \
2834 INSTR REG, (%rdi); \
2835 ret; \
2836 SET_SIZE(NAME)
2837
2838 SUWORD_NOERR(suword64_noerr, movq, %rsi)
2839 SUWORD_NOERR(suword32_noerr, movl, %esi)
2840 SUWORD_NOERR(suword16_noerr, movw, %si)
2841 SUWORD_NOERR(suword8_noerr, movb, %sil)
2842
2843 #elif defined(__i386)
2844
2845 #define SUWORD_NOERR(NAME, INSTR, REG) \
2846 ENTRY(NAME) \
2847 movl 4(%esp), %eax; \
2848 cmpl kernelbase, %eax; \
2849 jb 1f; \
2850 movl kernelbase, %eax; \
2851 1: \
2852 movl 8(%esp), %edx; \
2853 INSTR REG, (%eax); \
2854 ret; \
2855 SET_SIZE(NAME)
2856
2857 SUWORD_NOERR(suword32_noerr, movl, %edx)
2858 SUWORD_NOERR(suword16_noerr, movw, %dx)
2859 SUWORD_NOERR(suword8_noerr, movb, %dl)
2860
2861 #endif /* __i386 */
2862
2863 #undef SUWORD_NOERR
2864
2865 #endif /* lint */
2866
2867
2868 #if defined(__lint)
2869
2870 /*ARGSUSED*/
2871 int
2872 subyte(void *addr, uchar_t value)
2873 { return (0); }
2874
2875 /*ARGSUSED*/
2876 void
2877 subyte_noerr(void *addr, uchar_t value)
2878 {}
2879
2880 /*ARGSUSED*/
2881 int
2882 fulword(const void *addr, ulong_t *valuep)
2883 { return (0); }
2884
2885 /*ARGSUSED*/
2886 void
2887 fulword_noerr(const void *addr, ulong_t *valuep)
2888 {}
2889
2890 /*ARGSUSED*/
2891 int
2892 sulword(void *addr, ulong_t valuep)
2893 { return (0); }
2894
2895 /*ARGSUSED*/
2896 void
2897 sulword_noerr(void *addr, ulong_t valuep)
2898 {}
2899
2900 #else
2901
2902 .weak subyte
2903 subyte=suword8
2904 .weak subyte_noerr
2905 subyte_noerr=suword8_noerr
2906
2907 #if defined(__amd64)
2908
2909 .weak fulword
2910 fulword=fuword64
2911 .weak fulword_noerr
2912 fulword_noerr=fuword64_noerr
2913 .weak sulword
2914 sulword=suword64
2915 .weak sulword_noerr
2916 sulword_noerr=suword64_noerr
2917
2918 #elif defined(__i386)
2919
2920 .weak fulword
2921 fulword=fuword32
2922 .weak fulword_noerr
2923 fulword_noerr=fuword32_noerr
2924 .weak sulword
2925 sulword=suword32
2926 .weak sulword_noerr
2927 sulword_noerr=suword32_noerr
2928
2929 #endif /* __i386 */
2930
2931 #endif /* __lint */
2932
2933 #if defined(__lint)
2934
2935 /*
2936 * Copy a block of storage - must not overlap (from + len <= to).
2937 * No fault handler installed (to be called under on_fault())
2938 */
2939
2940 /* ARGSUSED */
2941 void
2942 copyout_noerr(const void *kfrom, void *uto, size_t count)
2943 {}
2944
2945 /* ARGSUSED */
2946 void
2947 copyin_noerr(const void *ufrom, void *kto, size_t count)
2948 {}
2949
2950 /*
2951 * Zero a block of storage in user space
2952 */
2953
2954 /* ARGSUSED */
2955 void
2956 uzero(void *addr, size_t count)
2957 {}
2958
2959 /*
2960 * copy a block of storage in user space
2961 */
2962
2963 /* ARGSUSED */
2964 void
2965 ucopy(const void *ufrom, void *uto, size_t ulength)
2966 {}
2967
2968 /*
2969 * copy a string in user space
2970 */
2971
2972 /* ARGSUSED */
2973 void
2974 ucopystr(const char *ufrom, char *uto, size_t umaxlength, size_t *lencopied)
2975 {}
2976
2977 #else /* __lint */
2978
2979 #if defined(__amd64)
2980
2981 ENTRY(copyin_noerr)
2982 movq kernelbase(%rip), %rax
2983 #ifdef DEBUG
2984 cmpq %rax, %rsi /* %rsi = kto */
2985 jae 1f
2986 leaq .cpyin_ne_pmsg(%rip), %rdi
2987 jmp call_panic /* setup stack and call panic */
2988 1:
2989 #endif
2990 cmpq %rax, %rdi /* ufrom < kernelbase */
2991 jb do_copy
2992 movq %rax, %rdi /* force fault at kernelbase */
2993 jmp do_copy
2994 SET_SIZE(copyin_noerr)
2995
2996 ENTRY(copyout_noerr)
2997 movq kernelbase(%rip), %rax
2998 #ifdef DEBUG
2999 cmpq %rax, %rdi /* %rdi = kfrom */
3000 jae 1f
3028 /*
3029 * Note, the frame pointer is required here becuase do_copystr expects
3030 * to be able to pop it off!
3031 */
3032 ENTRY(ucopystr)
3033 pushq %rbp
3034 movq %rsp, %rbp
3035 movq kernelbase(%rip), %rax
3036 cmpq %rax, %rdi
3037 cmovaeq %rax, %rdi /* force fault at kernelbase */
3038 cmpq %rax, %rsi
3039 cmovaeq %rax, %rsi /* force fault at kernelbase */
3040 /* do_copystr expects lofault address in %r8 */
3041 /* do_copystr expects whether or not we need smap in %r10 */
3042 xorl %r10d, %r10d
3043 movq %gs:CPU_THREAD, %r8
3044 movq T_LOFAULT(%r8), %r8
3045 jmp do_copystr
3046 SET_SIZE(ucopystr)
3047
3048 #elif defined(__i386)
3049
3050 ENTRY(copyin_noerr)
3051 movl kernelbase, %eax
3052 #ifdef DEBUG
3053 cmpl %eax, 8(%esp)
3054 jae 1f
3055 pushl $.cpyin_ne_pmsg
3056 call panic
3057 1:
3058 #endif
3059 cmpl %eax, 4(%esp)
3060 jb do_copy
3061 movl %eax, 4(%esp) /* force fault at kernelbase */
3062 jmp do_copy
3063 SET_SIZE(copyin_noerr)
3064
3065 ENTRY(copyout_noerr)
3066 movl kernelbase, %eax
3067 #ifdef DEBUG
3068 cmpl %eax, 4(%esp)
3069 jae 1f
3070 pushl $.cpyout_ne_pmsg
3071 call panic
3072 1:
3073 #endif
3074 cmpl %eax, 8(%esp)
3075 jb do_copy
3076 movl %eax, 8(%esp) /* force fault at kernelbase */
3077 jmp do_copy
3078 SET_SIZE(copyout_noerr)
3079
3080 ENTRY(uzero)
3081 movl kernelbase, %eax
3082 cmpl %eax, 4(%esp)
3083 jb do_zero
3084 movl %eax, 4(%esp) /* force fault at kernelbase */
3085 jmp do_zero
3086 SET_SIZE(uzero)
3087
3088 ENTRY(ucopy)
3089 movl kernelbase, %eax
3090 cmpl %eax, 4(%esp)
3091 jb 1f
3092 movl %eax, 4(%esp) /* force fault at kernelbase */
3093 1:
3094 cmpl %eax, 8(%esp)
3095 jb do_copy
3096 movl %eax, 8(%esp) /* force fault at kernelbase */
3097 jmp do_copy
3098 SET_SIZE(ucopy)
3099
3100 ENTRY(ucopystr)
3101 movl kernelbase, %eax
3102 cmpl %eax, 4(%esp)
3103 jb 1f
3104 movl %eax, 4(%esp) /* force fault at kernelbase */
3105 1:
3106 cmpl %eax, 8(%esp)
3107 jb 2f
3108 movl %eax, 8(%esp) /* force fault at kernelbase */
3109 2:
3110 /* do_copystr expects the lofault address in %eax */
3111 movl %gs:CPU_THREAD, %eax
3112 movl T_LOFAULT(%eax), %eax
3113 jmp do_copystr
3114 SET_SIZE(ucopystr)
3115
3116 #endif /* __i386 */
3117
3118 #ifdef DEBUG
3119 .data
3120 .kcopy_panic_msg:
3121 .string "kcopy: arguments below kernelbase"
3122 .bcopy_panic_msg:
3123 .string "bcopy: arguments below kernelbase"
3124 .kzero_panic_msg:
3125 .string "kzero: arguments below kernelbase"
3126 .bzero_panic_msg:
3127 .string "bzero: arguments below kernelbase"
3128 .copyin_panic_msg:
3129 .string "copyin: kaddr argument below kernelbase"
3130 .xcopyin_panic_msg:
3131 .string "xcopyin: kaddr argument below kernelbase"
3132 .copyout_panic_msg:
3133 .string "copyout: kaddr argument below kernelbase"
3134 .xcopyout_panic_msg:
3135 .string "xcopyout: kaddr argument below kernelbase"
3136 .copystr_panic_msg:
3137 .string "copystr: arguments in user space"
3138 .copyinstr_panic_msg:
3139 .string "copyinstr: kaddr argument not in kernel address space"
3140 .copyoutstr_panic_msg:
3141 .string "copyoutstr: kaddr argument not in kernel address space"
3142 .cpyin_ne_pmsg:
3143 .string "copyin_noerr: argument not in kernel address space"
3144 .cpyout_ne_pmsg:
3145 .string "copyout_noerr: argument not in kernel address space"
3146 #endif
3147
3148 #endif /* __lint */
3149
3150 /*
3151 * These functions are used for SMAP, supervisor mode access protection. They
3152 * are hotpatched to become real instructions when the system starts up which is
3153 * done in mlsetup() as a part of enabling the other CR4 related features.
3154 *
3155 * Generally speaking, smap_disable() is a stac instruction and smap_enable is a
3156 * clac instruction. It's safe to call these any number of times, and in fact,
3157 * out of paranoia, the kernel will likely call it at several points.
3158 */
3159
3160 #if defined(__lint)
3161
3162 void
3163 smap_enable(void)
3164 {}
3165
3166 void
3167 smap_disable(void)
3168 {}
3169
3170 #else
3171
3172 #if defined (__amd64) || defined(__i386)
3173 ENTRY(smap_disable)
3174 nop
3175 nop
3176 nop
3177 ret
3178 SET_SIZE(smap_disable)
3179
3180 ENTRY(smap_enable)
3181 nop
3182 nop
3183 nop
3184 ret
3185 SET_SIZE(smap_enable)
3186
3187 #endif /* __amd64 || __i386 */
3188
3189 #endif /* __lint */
3190
3191 #ifndef __lint
3192
3193 .data
3194 .align 4
3195 .globl _smap_enable_patch_count
3196 .type _smap_enable_patch_count,@object
3197 .size _smap_enable_patch_count, 4
3198 _smap_enable_patch_count:
3199 .long SMAP_ENABLE_COUNT
3200
3201 .globl _smap_disable_patch_count
3202 .type _smap_disable_patch_count,@object
3203 .size _smap_disable_patch_count, 4
3204 _smap_disable_patch_count:
3205 .long SMAP_DISABLE_COUNT
3206
3207 #endif /* __lint */
|
25
26 /*
27 * Copyright (c) 2009, Intel Corporation
28 * All rights reserved.
29 */
30
31 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
32 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
33 /* All Rights Reserved */
34
35 /* Copyright (c) 1987, 1988 Microsoft Corporation */
36 /* All Rights Reserved */
37
38 /*
39 * Copyright 2019 Joyent, Inc.
40 */
41
42 #include <sys/errno.h>
43 #include <sys/asm_linkage.h>
44
45 #include "assym.h"
46
47 #define KCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
48 #define XCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
49 /*
50 * Non-temopral access (NTA) alignment requirement
51 */
52 #define NTA_ALIGN_SIZE 4 /* Must be at least 4-byte aligned */
53 #define NTA_ALIGN_MASK _CONST(NTA_ALIGN_SIZE-1)
54 #define COUNT_ALIGN_SIZE 16 /* Must be at least 16-byte aligned */
55 #define COUNT_ALIGN_MASK _CONST(COUNT_ALIGN_SIZE-1)
56
57 /*
58 * With the introduction of Broadwell, Intel has introduced supervisor mode
59 * access protection -- SMAP. SMAP forces the kernel to set certain bits to
60 * enable access of user pages (AC in rflags, defines as PS_ACHK in
61 * <sys/psw.h>). One of the challenges is that the implementation of many of the
62 * userland copy routines directly use the kernel ones. For example, copyin and
63 * copyout simply go and jump to the do_copy_fault label and traditionally let
64 * those deal with the return for them. In fact, changing that is a can of frame
65 * pointers.
121 * The optimal 64-bit bcopy and kcopy for modern x86 processors uses
122 * "rep smovq" for large sizes. Performance data shows that many calls to
123 * bcopy/kcopy/bzero/kzero operate on small buffers. For best performance for
124 * these small sizes unrolled code is used. For medium sizes loops writing
125 * 64-bytes per loop are used. Transition points were determined experimentally.
126 */
127 #define BZERO_USE_REP (1024)
128 #define BCOPY_DFLT_REP (128)
129 #define BCOPY_NHM_REP (768)
130
131 /*
132 * Copy a block of storage, returning an error code if `from' or
133 * `to' takes a kernel pagefault which cannot be resolved.
134 * Returns errno value on pagefault error, 0 if all ok
135 */
136
137 /*
138 * I'm sorry about these macros, but copy.s is unsurprisingly sensitive to
139 * additional call instructions.
140 */
141 #define SMAP_DISABLE_COUNT 16
142 #define SMAP_ENABLE_COUNT 26
143
144 #define SMAP_DISABLE_INSTR(ITER) \
145 .globl _smap_disable_patch_/**/ITER; \
146 _smap_disable_patch_/**/ITER/**/:; \
147 nop; nop; nop;
148
149 #define SMAP_ENABLE_INSTR(ITER) \
150 .globl _smap_enable_patch_/**/ITER; \
151 _smap_enable_patch_/**/ITER/**/:; \
152 nop; nop; nop;
153
154 .globl kernelbase
155 .globl postbootkernelbase
156
157 ENTRY(kcopy)
158 pushq %rbp
159 movq %rsp, %rbp
160 #ifdef DEBUG
161 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
162 jb 0f
163 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
164 jnb 1f
165 0: leaq .kcopy_panic_msg(%rip), %rdi
166 xorl %eax, %eax
167 call panic
168 1:
169 #endif
170 /*
171 * pass lofault value as 4th argument to do_copy_fault
172 */
173 leaq _kcopy_copyerr(%rip), %rcx
174 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
175
176 do_copy_fault:
177 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
178 movq %rcx, T_LOFAULT(%r9) /* new lofault */
179 call bcopy_altentry
180 xorl %eax, %eax /* return 0 (success) */
181 SMAP_ENABLE_INSTR(0)
182
183 /*
184 * A fault during do_copy_fault is indicated through an errno value
185 * in %rax and we iretq from the trap handler to here.
186 */
187 _kcopy_copyerr:
188 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
189 leave
190 ret
191 SET_SIZE(kcopy)
192
193 #undef ARG_FROM
194 #undef ARG_TO
195 #undef ARG_COUNT
196
197 #define COPY_LOOP_INIT(src, dst, cnt) \
198 addq cnt, src; \
199 addq cnt, dst; \
200 shrq $3, cnt; \
201 neg cnt
202
203 /* Copy 16 bytes per loop. Uses %rax and %r8 */
204 #define COPY_LOOP_BODY(src, dst, cnt) \
205 prefetchnta 0x100(src, cnt, 8); \
206 movq (src, cnt, 8), %rax; \
207 movq 0x8(src, cnt, 8), %r8; \
208 movnti %rax, (dst, cnt, 8); \
209 movnti %r8, 0x8(dst, cnt, 8); \
210 addq $2, cnt
211
212 ENTRY(kcopy_nta)
213 pushq %rbp
214 movq %rsp, %rbp
215 #ifdef DEBUG
216 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
253 movq %rcx, T_LOFAULT(%r9) /* new lofault */
254
255 /*
256 * COPY_LOOP_BODY uses %rax and %r8
257 */
258 COPY_LOOP_INIT(%rdi, %rsi, %rdx)
259 2: COPY_LOOP_BODY(%rdi, %rsi, %rdx)
260 jnz 2b
261
262 mfence
263 xorl %eax, %eax /* return 0 (success) */
264 SMAP_ENABLE_INSTR(1)
265
266 _kcopy_nta_copyerr:
267 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
268 leave
269 ret
270 SET_SIZE(do_copy_fault_nta)
271 SET_SIZE(kcopy_nta)
272
273 ENTRY(bcopy)
274 #ifdef DEBUG
275 orq %rdx, %rdx /* %rdx = count */
276 jz 1f
277 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */
278 jb 0f
279 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */
280 jnb 1f
281 0: leaq .bcopy_panic_msg(%rip), %rdi
282 jmp call_panic /* setup stack and call panic */
283 1:
284 #endif
285 /*
286 * bcopy_altentry() is called from kcopy, i.e., do_copy_fault.
287 * kcopy assumes that bcopy doesn't touch %r9 and %r11. If bcopy
288 * uses these registers in future they must be saved and restored.
289 */
290 ALTENTRY(bcopy_altentry)
291 do_copy:
292 #define L(s) .bcopy/**/s
781
782 #ifdef DEBUG
783 /*
784 * Setup frame on the run-time stack. The end of the input argument
785 * area must be aligned on a 16 byte boundary. The stack pointer %rsp,
786 * always points to the end of the latest allocated stack frame.
787 * panic(const char *format, ...) is a varargs function. When a
788 * function taking variable arguments is called, %rax must be set
789 * to eight times the number of floating point parameters passed
790 * to the function in SSE registers.
791 */
792 call_panic:
793 pushq %rbp /* align stack properly */
794 movq %rsp, %rbp
795 xorl %eax, %eax /* no variable arguments */
796 call panic /* %rdi = format string */
797 #endif
798 SET_SIZE(bcopy_altentry)
799 SET_SIZE(bcopy)
800
801
802 /*
803 * Zero a block of storage, returning an error code if we
804 * take a kernel pagefault which cannot be resolved.
805 * Returns errno value on pagefault error, 0 if all ok
806 */
807
808 ENTRY(kzero)
809 #ifdef DEBUG
810 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */
811 jnb 0f
812 leaq .kzero_panic_msg(%rip), %rdi
813 jmp call_panic /* setup stack and call panic */
814 0:
815 #endif
816 /*
817 * pass lofault value as 3rd argument for fault return
818 */
819 leaq _kzeroerr(%rip), %rdx
820
821 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
822 movq T_LOFAULT(%r9), %r11 /* save the current lofault */
823 movq %rdx, T_LOFAULT(%r9) /* new lofault */
824 call bzero_altentry
825 xorl %eax, %eax
826 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
827 ret
828 /*
829 * A fault during bzero is indicated through an errno value
830 * in %rax when we iretq to here.
831 */
832 _kzeroerr:
833 addq $8, %rsp /* pop bzero_altentry call ret addr */
834 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
835 ret
836 SET_SIZE(kzero)
837
838 /*
839 * Zero a block of storage.
840 */
841
842 ENTRY(bzero)
843 #ifdef DEBUG
844 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */
845 jnb 0f
846 leaq .bzero_panic_msg(%rip), %rdi
847 jmp call_panic /* setup stack and call panic */
848 0:
849 #endif
850 ALTENTRY(bzero_altentry)
851 do_zero:
852 #define L(s) .bzero/**/s
853 xorl %eax, %eax
854
855 cmpq $0x50, %rsi /* 80 */
856 jae L(ck_align)
857
858 /*
859 * Performance data shows many caller's are zeroing small buffers. So
860 * for best perf for these sizes unrolled code is used. Store zeros
861 * without worrying about alignment.
1136 movslq (%r10,%rsi,4), %rcx
1137 leaq (%rcx,%r10,1), %r10
1138 INDIRECT_JMP_REG(r10)
1139
1140 /*
1141 * Use rep sstoq. Clear any remainder via unrolled code
1142 */
1143 .p2align 4
1144 L(use_rep):
1145 movq %rsi, %rcx /* get size in bytes */
1146 shrq $3, %rcx /* count of 8-byte words to zero */
1147 rep
1148 sstoq /* %rcx = words to clear (%rax=0) */
1149 andq $7, %rsi /* remaining bytes */
1150 jnz 9b
1151 ret
1152 #undef L
1153 SET_SIZE(bzero_altentry)
1154 SET_SIZE(bzero)
1155
1156 /*
1157 * Transfer data to and from user space -
1158 * Note that these routines can cause faults
1159 * It is assumed that the kernel has nothing at
1160 * less than KERNELBASE in the virtual address space.
1161 *
1162 * Note that copyin(9F) and copyout(9F) are part of the
1163 * DDI/DKI which specifies that they return '-1' on "errors."
1164 *
1165 * Sigh.
1166 *
1167 * So there's two extremely similar routines - xcopyin_nta() and
1168 * xcopyout_nta() which return the errno that we've faithfully computed.
1169 * This allows other callers (e.g. uiomove(9F)) to work correctly.
1170 * Given that these are used pretty heavily, we expand the calling
1171 * sequences inline for all flavours (rather than making wrappers).
1172 */
1173
1174 /*
1175 * Copy user data to kernel space.
1176 */
1177
1178 ENTRY(copyin)
1179 pushq %rbp
1180 movq %rsp, %rbp
1181 subq $24, %rsp
1182
1183 /*
1184 * save args in case we trap and need to rerun as a copyop
1185 */
1186 movq %rdi, (%rsp)
1187 movq %rsi, 0x8(%rsp)
1188 movq %rdx, 0x10(%rsp)
1189
1190 movq kernelbase(%rip), %rax
1191 #ifdef DEBUG
1192 cmpq %rax, %rsi /* %rsi = kaddr */
1193 jnb 1f
1194 leaq .copyin_panic_msg(%rip), %rdi
1195 xorl %eax, %eax
1196 call panic
1197 1:
1213 addq $8, %rsp /* pop bcopy_altentry call ret addr */
1214 3:
1215 movq T_COPYOPS(%r9), %rax
1216 cmpq $0, %rax
1217 jz 2f
1218 /*
1219 * reload args for the copyop
1220 */
1221 movq (%rsp), %rdi
1222 movq 0x8(%rsp), %rsi
1223 movq 0x10(%rsp), %rdx
1224 leave
1225 movq CP_COPYIN(%rax), %rax
1226 INDIRECT_JMP_REG(rax)
1227
1228 2: movl $-1, %eax
1229 leave
1230 ret
1231 SET_SIZE(copyin)
1232
1233 ENTRY(xcopyin_nta)
1234 pushq %rbp
1235 movq %rsp, %rbp
1236 subq $24, %rsp
1237
1238 /*
1239 * save args in case we trap and need to rerun as a copyop
1240 * %rcx is consumed in this routine so we don't need to save
1241 * it.
1242 */
1243 movq %rdi, (%rsp)
1244 movq %rsi, 0x8(%rsp)
1245 movq %rdx, 0x10(%rsp)
1246
1247 movq kernelbase(%rip), %rax
1248 #ifdef DEBUG
1249 cmpq %rax, %rsi /* %rsi = kaddr */
1250 jnb 1f
1251 leaq .xcopyin_panic_msg(%rip), %rdi
1252 xorl %eax, %eax
1302 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
1303 3:
1304 movq T_COPYOPS(%r9), %r8
1305 cmpq $0, %r8
1306 jz 2f
1307
1308 /*
1309 * reload args for the copyop
1310 */
1311 movq (%rsp), %rdi
1312 movq 0x8(%rsp), %rsi
1313 movq 0x10(%rsp), %rdx
1314 leave
1315 movq CP_XCOPYIN(%r8), %r8
1316 INDIRECT_JMP_REG(r8)
1317
1318 2: leave
1319 ret
1320 SET_SIZE(xcopyin_nta)
1321
1322 /*
1323 * Copy kernel data to user space.
1324 */
1325
1326 ENTRY(copyout)
1327 pushq %rbp
1328 movq %rsp, %rbp
1329 subq $24, %rsp
1330
1331 /*
1332 * save args in case we trap and need to rerun as a copyop
1333 */
1334 movq %rdi, (%rsp)
1335 movq %rsi, 0x8(%rsp)
1336 movq %rdx, 0x10(%rsp)
1337
1338 movq kernelbase(%rip), %rax
1339 #ifdef DEBUG
1340 cmpq %rax, %rdi /* %rdi = kaddr */
1341 jnb 1f
1342 leaq .copyout_panic_msg(%rip), %rdi
1343 xorl %eax, %eax
1344 call panic
1345 1:
1362 3:
1363 movq T_COPYOPS(%r9), %rax
1364 cmpq $0, %rax
1365 jz 2f
1366
1367 /*
1368 * reload args for the copyop
1369 */
1370 movq (%rsp), %rdi
1371 movq 0x8(%rsp), %rsi
1372 movq 0x10(%rsp), %rdx
1373 leave
1374 movq CP_COPYOUT(%rax), %rax
1375 INDIRECT_JMP_REG(rax)
1376
1377 2: movl $-1, %eax
1378 leave
1379 ret
1380 SET_SIZE(copyout)
1381
1382 ENTRY(xcopyout_nta)
1383 pushq %rbp
1384 movq %rsp, %rbp
1385 subq $24, %rsp
1386
1387 /*
1388 * save args in case we trap and need to rerun as a copyop
1389 */
1390 movq %rdi, (%rsp)
1391 movq %rsi, 0x8(%rsp)
1392 movq %rdx, 0x10(%rsp)
1393
1394 movq kernelbase(%rip), %rax
1395 #ifdef DEBUG
1396 cmpq %rax, %rdi /* %rdi = kaddr */
1397 jnb 1f
1398 leaq .xcopyout_panic_msg(%rip), %rdi
1399 xorl %eax, %eax
1400 call panic
1401 1:
1452 movq %r11, T_LOFAULT(%r9) /* restore original lofault */
1453 3:
1454 movq T_COPYOPS(%r9), %r8
1455 cmpq $0, %r8
1456 jz 2f
1457
1458 /*
1459 * reload args for the copyop
1460 */
1461 movq (%rsp), %rdi
1462 movq 0x8(%rsp), %rsi
1463 movq 0x10(%rsp), %rdx
1464 leave
1465 movq CP_XCOPYOUT(%r8), %r8
1466 INDIRECT_JMP_REG(r8)
1467
1468 2: leave
1469 ret
1470 SET_SIZE(xcopyout_nta)
1471
1472 /*
1473 * Copy a null terminated string from one point to another in
1474 * the kernel address space.
1475 */
1476
1477 ENTRY(copystr)
1478 pushq %rbp
1479 movq %rsp, %rbp
1480 #ifdef DEBUG
1481 movq kernelbase(%rip), %rax
1482 cmpq %rax, %rdi /* %rdi = from */
1483 jb 0f
1484 cmpq %rax, %rsi /* %rsi = to */
1485 jnb 1f
1486 0: leaq .copystr_panic_msg(%rip), %rdi
1487 xorl %eax, %eax
1488 call panic
1489 1:
1490 #endif
1491 movq %gs:CPU_THREAD, %r9
1492 movq T_LOFAULT(%r9), %r8 /* pass current lofault value as */
1493 /* 5th argument to do_copystr */
1494 xorl %r10d,%r10d /* pass smap restore need in %r10d */
1495 /* as a non-ABI 6th arg */
1496 do_copystr:
1521 copystr_null:
1522 xorl %eax, %eax /* no error */
1523
1524 copystr_out:
1525 cmpq $0, %rcx /* want length? */
1526 je copystr_smap /* no */
1527 subq %r8, %rdx /* compute length and store it */
1528 movq %rdx, (%rcx)
1529
1530 copystr_smap:
1531 cmpl $0, %r10d
1532 jz copystr_done
1533 SMAP_ENABLE_INSTR(7)
1534
1535 copystr_done:
1536 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
1537 leave
1538 ret
1539 SET_SIZE(copystr)
1540
1541 /*
1542 * Copy a null terminated string from the user address space into
1543 * the kernel address space.
1544 */
1545
1546 ENTRY(copyinstr)
1547 pushq %rbp
1548 movq %rsp, %rbp
1549 subq $32, %rsp
1550
1551 /*
1552 * save args in case we trap and need to rerun as a copyop
1553 */
1554 movq %rdi, (%rsp)
1555 movq %rsi, 0x8(%rsp)
1556 movq %rdx, 0x10(%rsp)
1557 movq %rcx, 0x18(%rsp)
1558
1559 movq kernelbase(%rip), %rax
1560 #ifdef DEBUG
1561 cmpq %rax, %rsi /* %rsi = kaddr */
1562 jnb 1f
1563 leaq .copyinstr_panic_msg(%rip), %rdi
1564 xorl %eax, %eax
1565 call panic
1587 movq T_COPYOPS(%r9), %rax
1588 cmpq $0, %rax
1589 jz 2f
1590
1591 /*
1592 * reload args for the copyop
1593 */
1594 movq (%rsp), %rdi
1595 movq 0x8(%rsp), %rsi
1596 movq 0x10(%rsp), %rdx
1597 movq 0x18(%rsp), %rcx
1598 leave
1599 movq CP_COPYINSTR(%rax), %rax
1600 INDIRECT_JMP_REG(rax)
1601
1602 2: movl $EFAULT, %eax /* return EFAULT */
1603 leave
1604 ret
1605 SET_SIZE(copyinstr)
1606
1607 /*
1608 * Copy a null terminated string from the kernel
1609 * address space to the user address space.
1610 */
1611
1612 ENTRY(copyoutstr)
1613 pushq %rbp
1614 movq %rsp, %rbp
1615 subq $32, %rsp
1616
1617 /*
1618 * save args in case we trap and need to rerun as a copyop
1619 */
1620 movq %rdi, (%rsp)
1621 movq %rsi, 0x8(%rsp)
1622 movq %rdx, 0x10(%rsp)
1623 movq %rcx, 0x18(%rsp)
1624
1625 movq kernelbase(%rip), %rax
1626 #ifdef DEBUG
1627 cmpq %rax, %rdi /* %rdi = kaddr */
1628 jnb 1f
1629 leaq .copyoutstr_panic_msg(%rip), %rdi
1630 jmp call_panic /* setup stack and call panic */
1631 1:
1652 movq T_COPYOPS(%r9), %rax
1653 cmpq $0, %rax
1654 jz 2f
1655
1656 /*
1657 * reload args for the copyop
1658 */
1659 movq (%rsp), %rdi
1660 movq 0x8(%rsp), %rsi
1661 movq 0x10(%rsp), %rdx
1662 movq 0x18(%rsp), %rcx
1663 leave
1664 movq CP_COPYOUTSTR(%rax), %rax
1665 INDIRECT_JMP_REG(rax)
1666
1667 2: movl $EFAULT, %eax /* return EFAULT */
1668 leave
1669 ret
1670 SET_SIZE(copyoutstr)
1671
1672 /*
1673 * Since all of the fuword() variants are so similar, we have a macro to spit
1674 * them out. This allows us to create DTrace-unobservable functions easily.
1675 */
1676
1677 /*
1678 * Note that we don't save and reload the arguments here
1679 * because their values are not altered in the copy path.
1680 * Additionally, when successful, the smap_enable jmp will
1681 * actually return us to our original caller.
1682 */
1683
1684 #define FUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2) \
1685 ENTRY(NAME) \
1686 movq %gs:CPU_THREAD, %r9; \
1687 cmpq kernelbase(%rip), %rdi; \
1688 jae 1f; \
1689 leaq _flt_/**/NAME, %rdx; \
1690 movq %rdx, T_LOFAULT(%r9); \
1691 SMAP_DISABLE_INSTR(DISNUM) \
1692 INSTR (%rdi), REG; \
1693 movq $0, T_LOFAULT(%r9); \
1694 INSTR REG, (%rsi); \
1695 xorl %eax, %eax; \
1696 SMAP_ENABLE_INSTR(EN1) \
1697 ret; \
1698 _flt_/**/NAME: \
1699 SMAP_ENABLE_INSTR(EN2) \
1700 movq $0, T_LOFAULT(%r9); \
1701 1: \
1702 movq T_COPYOPS(%r9), %rax; \
1703 cmpq $0, %rax; \
1704 jz 2f; \
1705 movq COPYOP(%rax), %rax; \
1706 INDIRECT_JMP_REG(rax); \
1707 2: \
1708 movl $-1, %eax; \
1709 ret; \
1710 SET_SIZE(NAME)
1711
1712 FUWORD(fuword64, movq, %rax, CP_FUWORD64,8,10,11)
1713 FUWORD(fuword32, movl, %eax, CP_FUWORD32,9,12,13)
1714 FUWORD(fuword16, movw, %ax, CP_FUWORD16,10,14,15)
1715 FUWORD(fuword8, movb, %al, CP_FUWORD8,11,16,17)
1716
1717 #undef FUWORD
1718
1719 /*
1720 * Set user word.
1721 */
1722
1723 /*
1724 * Note that we don't save and reload the arguments here
1725 * because their values are not altered in the copy path.
1726 */
1727
1728 #define SUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2) \
1729 ENTRY(NAME) \
1730 movq %gs:CPU_THREAD, %r9; \
1731 cmpq kernelbase(%rip), %rdi; \
1732 jae 1f; \
1733 leaq _flt_/**/NAME, %rdx; \
1734 SMAP_DISABLE_INSTR(DISNUM) \
1735 movq %rdx, T_LOFAULT(%r9); \
1736 INSTR REG, (%rdi); \
1737 movq $0, T_LOFAULT(%r9); \
1738 xorl %eax, %eax; \
1739 SMAP_ENABLE_INSTR(EN1) \
1740 ret; \
1741 _flt_/**/NAME: \
1742 SMAP_ENABLE_INSTR(EN2) \
1743 movq $0, T_LOFAULT(%r9); \
1744 1: \
1745 movq T_COPYOPS(%r9), %rax; \
1746 cmpq $0, %rax; \
1747 jz 3f; \
1748 movq COPYOP(%rax), %rax; \
1749 INDIRECT_JMP_REG(rax); \
1750 3: \
1751 movl $-1, %eax; \
1752 ret; \
1753 SET_SIZE(NAME)
1754
1755 SUWORD(suword64, movq, %rsi, CP_SUWORD64,12,18,19)
1756 SUWORD(suword32, movl, %esi, CP_SUWORD32,13,20,21)
1757 SUWORD(suword16, movw, %si, CP_SUWORD16,14,22,23)
1758 SUWORD(suword8, movb, %sil, CP_SUWORD8,15,24,25)
1759
1760 #undef SUWORD
1761
1762 #define FUWORD_NOERR(NAME, INSTR, REG) \
1763 ENTRY(NAME) \
1764 cmpq kernelbase(%rip), %rdi; \
1765 cmovnbq kernelbase(%rip), %rdi; \
1766 INSTR (%rdi), REG; \
1767 INSTR REG, (%rsi); \
1768 ret; \
1769 SET_SIZE(NAME)
1770
1771 FUWORD_NOERR(fuword64_noerr, movq, %rax)
1772 FUWORD_NOERR(fuword32_noerr, movl, %eax)
1773 FUWORD_NOERR(fuword16_noerr, movw, %ax)
1774 FUWORD_NOERR(fuword8_noerr, movb, %al)
1775
1776 #undef FUWORD_NOERR
1777
1778 #define SUWORD_NOERR(NAME, INSTR, REG) \
1779 ENTRY(NAME) \
1780 cmpq kernelbase(%rip), %rdi; \
1781 cmovnbq kernelbase(%rip), %rdi; \
1782 INSTR REG, (%rdi); \
1783 ret; \
1784 SET_SIZE(NAME)
1785
1786 SUWORD_NOERR(suword64_noerr, movq, %rsi)
1787 SUWORD_NOERR(suword32_noerr, movl, %esi)
1788 SUWORD_NOERR(suword16_noerr, movw, %si)
1789 SUWORD_NOERR(suword8_noerr, movb, %sil)
1790
1791 #undef SUWORD_NOERR
1792
1793
1794 .weak subyte
1795 subyte=suword8
1796 .weak subyte_noerr
1797 subyte_noerr=suword8_noerr
1798
1799 .weak fulword
1800 fulword=fuword64
1801 .weak fulword_noerr
1802 fulword_noerr=fuword64_noerr
1803 .weak sulword
1804 sulword=suword64
1805 .weak sulword_noerr
1806 sulword_noerr=suword64_noerr
1807
1808 ENTRY(copyin_noerr)
1809 movq kernelbase(%rip), %rax
1810 #ifdef DEBUG
1811 cmpq %rax, %rsi /* %rsi = kto */
1812 jae 1f
1813 leaq .cpyin_ne_pmsg(%rip), %rdi
1814 jmp call_panic /* setup stack and call panic */
1815 1:
1816 #endif
1817 cmpq %rax, %rdi /* ufrom < kernelbase */
1818 jb do_copy
1819 movq %rax, %rdi /* force fault at kernelbase */
1820 jmp do_copy
1821 SET_SIZE(copyin_noerr)
1822
1823 ENTRY(copyout_noerr)
1824 movq kernelbase(%rip), %rax
1825 #ifdef DEBUG
1826 cmpq %rax, %rdi /* %rdi = kfrom */
1827 jae 1f
1855 /*
1856 * Note, the frame pointer is required here becuase do_copystr expects
1857 * to be able to pop it off!
1858 */
1859 ENTRY(ucopystr)
1860 pushq %rbp
1861 movq %rsp, %rbp
1862 movq kernelbase(%rip), %rax
1863 cmpq %rax, %rdi
1864 cmovaeq %rax, %rdi /* force fault at kernelbase */
1865 cmpq %rax, %rsi
1866 cmovaeq %rax, %rsi /* force fault at kernelbase */
1867 /* do_copystr expects lofault address in %r8 */
1868 /* do_copystr expects whether or not we need smap in %r10 */
1869 xorl %r10d, %r10d
1870 movq %gs:CPU_THREAD, %r8
1871 movq T_LOFAULT(%r8), %r8
1872 jmp do_copystr
1873 SET_SIZE(ucopystr)
1874
1875 #ifdef DEBUG
1876 .data
1877 .kcopy_panic_msg:
1878 .string "kcopy: arguments below kernelbase"
1879 .bcopy_panic_msg:
1880 .string "bcopy: arguments below kernelbase"
1881 .kzero_panic_msg:
1882 .string "kzero: arguments below kernelbase"
1883 .bzero_panic_msg:
1884 .string "bzero: arguments below kernelbase"
1885 .copyin_panic_msg:
1886 .string "copyin: kaddr argument below kernelbase"
1887 .xcopyin_panic_msg:
1888 .string "xcopyin: kaddr argument below kernelbase"
1889 .copyout_panic_msg:
1890 .string "copyout: kaddr argument below kernelbase"
1891 .xcopyout_panic_msg:
1892 .string "xcopyout: kaddr argument below kernelbase"
1893 .copystr_panic_msg:
1894 .string "copystr: arguments in user space"
1895 .copyinstr_panic_msg:
1896 .string "copyinstr: kaddr argument not in kernel address space"
1897 .copyoutstr_panic_msg:
1898 .string "copyoutstr: kaddr argument not in kernel address space"
1899 .cpyin_ne_pmsg:
1900 .string "copyin_noerr: argument not in kernel address space"
1901 .cpyout_ne_pmsg:
1902 .string "copyout_noerr: argument not in kernel address space"
1903 #endif
1904
1905 /*
1906 * These functions are used for SMAP, supervisor mode access protection. They
1907 * are hotpatched to become real instructions when the system starts up which is
1908 * done in mlsetup() as a part of enabling the other CR4 related features.
1909 *
1910 * Generally speaking, smap_disable() is a stac instruction and smap_enable is a
1911 * clac instruction. It's safe to call these any number of times, and in fact,
1912 * out of paranoia, the kernel will likely call it at several points.
1913 */
1914
1915 ENTRY(smap_disable)
1916 nop
1917 nop
1918 nop
1919 ret
1920 SET_SIZE(smap_disable)
1921
1922 ENTRY(smap_enable)
1923 nop
1924 nop
1925 nop
1926 ret
1927 SET_SIZE(smap_enable)
1928
1929 .data
1930 .align 4
1931 .globl _smap_enable_patch_count
1932 .type _smap_enable_patch_count,@object
1933 .size _smap_enable_patch_count, 4
1934 _smap_enable_patch_count:
1935 .long SMAP_ENABLE_COUNT
1936
1937 .globl _smap_disable_patch_count
1938 .type _smap_disable_patch_count,@object
1939 .size _smap_disable_patch_count, 4
1940 _smap_disable_patch_count:
1941 .long SMAP_DISABLE_COUNT
|