25 */
26
27 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
29 /* All Rights Reserved */
30
31 /* Copyright (c) 1987, 1988 Microsoft Corporation */
32 /* All Rights Reserved */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 #include <sys/asm_linkage.h>
40 #include <sys/asm_misc.h>
41 #include <sys/regset.h>
42 #include <sys/privregs.h>
43 #include <sys/x86_archext.h>
44
45 #if defined(__lint)
46 #include <sys/types.h>
47 #include <sys/fp.h>
48 #else
49 #include "assym.h"
50 #endif
51
52 #if defined(__lint)
53
54 uint_t
55 fpu_initial_probe(void)
56 { return (0); }
57
58 #else /* __lint */
59
60 /*
61 * Returns zero if x87 "chip" is present(!)
62 */
63 ENTRY_NP(fpu_initial_probe)
64 CLTS
65 fninit
66 fnstsw %ax
67 movzbl %al, %eax
68 ret
69 SET_SIZE(fpu_initial_probe)
70
71 #endif /* __lint */
72
73 #if defined(__lint)
74
75 /*ARGSUSED*/
76 void
77 fxsave_insn(struct fxsave_state *fx)
78 {}
79
80 #else /* __lint */
81
82 ENTRY_NP(fxsave_insn)
83 fxsaveq (%rdi)
84 ret
85 SET_SIZE(fxsave_insn)
86
87 #endif /* __lint */
88
89 /*
90 * One of these routines is called from any lwp with floating
91 * point context as part of the prolog of a context switch.
92 */
93
94 #if defined(__lint)
95
96 /*ARGSUSED*/
97 void
98 xsave_ctxt(void *arg)
99 {}
100
101 /*ARGSUSED*/
102 void
103 xsaveopt_ctxt(void *arg)
104 {}
105
106 /*ARGSUSED*/
107 void
108 fpxsave_ctxt(void *arg)
109 {}
110
111 #else /* __lint */
112
113 /*
114 * These three functions define the Intel "xsave" handling for CPUs with
115 * different features. Newer AMD CPUs can also use these functions. See the
116 * 'exception pointers' comment below.
117 */
118 ENTRY_NP(fpxsave_ctxt) /* %rdi is a struct fpu_ctx */
119 cmpl $FPU_EN, FPU_CTX_FPU_FLAGS(%rdi)
120 jne 1f
121 movl $_CONST(FPU_VALID|FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
122 movq FPU_CTX_FPU_REGS(%rdi), %rdi /* fpu_regs.kfpu_u.kfpu_fx ptr */
123 fxsaveq (%rdi)
124 STTS(%rsi) /* trap on next fpu touch */
125 1: rep; ret /* use 2 byte return instruction when branch target */
126 /* AMD Software Optimization Guide - Section 6.2 */
127 SET_SIZE(fpxsave_ctxt)
128
129 ENTRY_NP(xsave_ctxt)
130 cmpl $FPU_EN, FPU_CTX_FPU_FLAGS(%rdi)
131 jne 1f
132 movl $_CONST(FPU_VALID|FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
207 jne 1f
208 movl $_CONST(FPU_VALID|FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
209 movl FPU_CTX_FPU_XSAVE_MASK(%rdi), %eax
210 movl FPU_CTX_FPU_XSAVE_MASK+4(%rdi), %edx
211 movq FPU_CTX_FPU_REGS(%rdi), %rsi /* fpu_regs.kfpu_u.kfpu_xs ptr */
212 xsaveopt (%rsi)
213 btw $7, FXSAVE_STATE_FSW(%rsi) /* Test saved ES bit */
214 jnc 0f /* jump if ES = 0 */
215 fnclex /* clear pending x87 exceptions */
216 0: ffree %st(7) /* clear tag bit to remove possible stack overflow */
217 fildl .fpzero_const(%rip) /* dummy load changes all excp. pointers */
218 STTS(%rsi) /* trap on next fpu touch */
219 1: ret
220 SET_SIZE(xsaveopt_excp_clr_ctxt)
221
222 .align 8
223 .fpzero_const:
224 .4byte 0x0
225 .4byte 0x0
226
227 #endif /* __lint */
228
229
230 #if defined(__lint)
231
232 /*ARGSUSED*/
233 void
234 fpsave(struct fnsave_state *f)
235 {}
236
237 /*ARGSUSED*/
238 void
239 fpxsave(struct fxsave_state *f)
240 {}
241
242 /*ARGSUSED*/
243 void
244 xsave(struct xsave_state *f, uint64_t m)
245 {}
246
247 /*ARGSUSED*/
248 void
249 xsaveopt(struct xsave_state *f, uint64_t m)
250 {}
251
252 #else /* __lint */
253
254 ENTRY_NP(fpxsave)
255 CLTS
256 fxsaveq (%rdi)
257 fninit /* clear exceptions, init x87 tags */
258 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
259 ret
260 SET_SIZE(fpxsave)
261
262 ENTRY_NP(xsave)
263 CLTS
264 movl %esi, %eax /* bv mask */
265 movq %rsi, %rdx
266 shrq $32, %rdx
267 xsave (%rdi)
268
269 fninit /* clear exceptions, init x87 tags */
270 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
271 ret
272 SET_SIZE(xsave)
273
274 ENTRY_NP(xsaveopt)
275 CLTS
276 movl %esi, %eax /* bv mask */
277 movq %rsi, %rdx
278 shrq $32, %rdx
279 xsaveopt (%rdi)
280
281 fninit /* clear exceptions, init x87 tags */
282 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
283 ret
284 SET_SIZE(xsaveopt)
285
286 #endif /* __lint */
287
288 /*
289 * These functions are used when restoring the FPU as part of the epilogue of a
290 * context switch.
291 */
292
293 #if defined(__lint)
294
295 /*ARGSUSED*/
296 void
297 fpxrestore_ctxt(void *arg)
298 {}
299
300 /*ARGSUSED*/
301 void
302 xrestore_ctxt(void *arg)
303 {}
304
305 #else /* __lint */
306
307 ENTRY(fpxrestore_ctxt)
308 cmpl $_CONST(FPU_EN|FPU_VALID), FPU_CTX_FPU_FLAGS(%rdi)
309 jne 1f
310 movl $_CONST(FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
311 movq FPU_CTX_FPU_REGS(%rdi), %rdi /* fpu_regs.kfpu_u.kfpu_fx ptr */
312 CLTS
313 fxrstorq (%rdi)
314 1:
315 ret
316 SET_SIZE(fpxrestore_ctxt)
317
318 ENTRY(xrestore_ctxt)
319 cmpl $_CONST(FPU_EN|FPU_VALID), FPU_CTX_FPU_FLAGS(%rdi)
320 jne 1f
321 movl $_CONST(FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
322 movl FPU_CTX_FPU_XSAVE_MASK(%rdi), %eax /* xsave flags in EDX:EAX */
323 movl FPU_CTX_FPU_XSAVE_MASK+4(%rdi), %edx
324 movq FPU_CTX_FPU_REGS(%rdi), %rdi /* fpu_regs.kfpu_u.kfpu_xs ptr */
325 CLTS
326 xrstor (%rdi)
327 1:
328 ret
329 SET_SIZE(xrestore_ctxt)
330
331 #endif /* __lint */
332
333
334 #if defined(__lint)
335
336 /*ARGSUSED*/
337 void
338 fpxrestore(struct fxsave_state *f)
339 {}
340
341 /*ARGSUSED*/
342 void
343 xrestore(struct xsave_state *f, uint64_t m)
344 {}
345
346 #else /* __lint */
347
348 ENTRY_NP(fpxrestore)
349 CLTS
350 fxrstorq (%rdi)
351 ret
352 SET_SIZE(fpxrestore)
353
354 ENTRY_NP(xrestore)
355 CLTS
356 movl %esi, %eax /* bv mask */
357 movq %rsi, %rdx
358 shrq $32, %rdx
359 xrstor (%rdi)
360 ret
361 SET_SIZE(xrestore)
362
363 #endif /* __lint */
364
365 /*
366 * Disable the floating point unit.
367 */
368
369 #if defined(__lint)
370
371 void
372 fpdisable(void)
373 {}
374
375 #else /* __lint */
376
377 ENTRY_NP(fpdisable)
378 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
379 ret
380 SET_SIZE(fpdisable)
381
382 #endif /* __lint */
383
384 /*
385 * Initialize the fpu hardware.
386 */
387
388 #if defined(__lint)
389
390 void
391 fpinit(void)
392 {}
393
394 #else /* __lint */
395
396 ENTRY_NP(fpinit)
397 CLTS
398 cmpl $FP_XSAVE, fp_save_mech
399 je 1f
400
401 /* fxsave */
402 leaq sse_initial(%rip), %rax
403 fxrstorq (%rax) /* load clean initial state */
404 ret
405
406 1: /* xsave */
407 leaq avx_initial(%rip), %rcx
408 xorl %edx, %edx
409 movl $XFEATURE_AVX, %eax
410 bt $X86FSET_AVX, x86_featureset
411 cmovael %edx, %eax
412 orl $(XFEATURE_LEGACY_FP | XFEATURE_SSE), %eax
413 xrstor (%rcx)
414 ret
415 SET_SIZE(fpinit)
416
417 #endif /* __lint */
418
419 /*
420 * Clears FPU exception state.
421 * Returns the FP status word.
422 */
423
424 #if defined(__lint)
425
426 uint32_t
427 fperr_reset(void)
428 { return (0); }
429
430 uint32_t
431 fpxerr_reset(void)
432 { return (0); }
433
434 #else /* __lint */
435
436 ENTRY_NP(fperr_reset)
437 CLTS
438 xorl %eax, %eax
439 fnstsw %ax
440 fnclex
441 ret
442 SET_SIZE(fperr_reset)
443
444 ENTRY_NP(fpxerr_reset)
445 pushq %rbp
446 movq %rsp, %rbp
447 subq $0x10, %rsp /* make some temporary space */
448 CLTS
449 stmxcsr (%rsp)
450 movl (%rsp), %eax
451 andl $_BITNOT(SSE_MXCSR_EFLAGS), (%rsp)
452 ldmxcsr (%rsp) /* clear processor exceptions */
453 leave
454 ret
455 SET_SIZE(fpxerr_reset)
456
457 #endif /* __lint */
458
459 #if defined(__lint)
460
461 uint32_t
462 fpgetcwsw(void)
463 {
464 return (0);
465 }
466
467 #else /* __lint */
468
469 ENTRY_NP(fpgetcwsw)
470 pushq %rbp
471 movq %rsp, %rbp
472 subq $0x10, %rsp /* make some temporary space */
473 CLTS
474 fnstsw (%rsp) /* store the status word */
475 fnstcw 2(%rsp) /* store the control word */
476 movl (%rsp), %eax /* put both in %eax */
477 leave
478 ret
479 SET_SIZE(fpgetcwsw)
480
481 #endif /* __lint */
482
483 /*
484 * Returns the MXCSR register.
485 */
486
487 #if defined(__lint)
488
489 uint32_t
490 fpgetmxcsr(void)
491 {
492 return (0);
493 }
494
495 #else /* __lint */
496
497 ENTRY_NP(fpgetmxcsr)
498 pushq %rbp
499 movq %rsp, %rbp
500 subq $0x10, %rsp /* make some temporary space */
501 CLTS
502 stmxcsr (%rsp)
503 movl (%rsp), %eax
504 leave
505 ret
506 SET_SIZE(fpgetmxcsr)
507
508 #endif /* __lint */
|
25 */
26
27 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
29 /* All Rights Reserved */
30
31 /* Copyright (c) 1987, 1988 Microsoft Corporation */
32 /* All Rights Reserved */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 #include <sys/asm_linkage.h>
40 #include <sys/asm_misc.h>
41 #include <sys/regset.h>
42 #include <sys/privregs.h>
43 #include <sys/x86_archext.h>
44
45 #include "assym.h"
46
47 /*
48 * Returns zero if x87 "chip" is present(!)
49 */
50 ENTRY_NP(fpu_initial_probe)
51 CLTS
52 fninit
53 fnstsw %ax
54 movzbl %al, %eax
55 ret
56 SET_SIZE(fpu_initial_probe)
57
58 ENTRY_NP(fxsave_insn)
59 fxsaveq (%rdi)
60 ret
61 SET_SIZE(fxsave_insn)
62
63 /*
64 * One of these routines is called from any lwp with floating
65 * point context as part of the prolog of a context switch.
66 */
67
68 /*
69 * These three functions define the Intel "xsave" handling for CPUs with
70 * different features. Newer AMD CPUs can also use these functions. See the
71 * 'exception pointers' comment below.
72 */
73 ENTRY_NP(fpxsave_ctxt) /* %rdi is a struct fpu_ctx */
74 cmpl $FPU_EN, FPU_CTX_FPU_FLAGS(%rdi)
75 jne 1f
76 movl $_CONST(FPU_VALID|FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
77 movq FPU_CTX_FPU_REGS(%rdi), %rdi /* fpu_regs.kfpu_u.kfpu_fx ptr */
78 fxsaveq (%rdi)
79 STTS(%rsi) /* trap on next fpu touch */
80 1: rep; ret /* use 2 byte return instruction when branch target */
81 /* AMD Software Optimization Guide - Section 6.2 */
82 SET_SIZE(fpxsave_ctxt)
83
84 ENTRY_NP(xsave_ctxt)
85 cmpl $FPU_EN, FPU_CTX_FPU_FLAGS(%rdi)
86 jne 1f
87 movl $_CONST(FPU_VALID|FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
162 jne 1f
163 movl $_CONST(FPU_VALID|FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
164 movl FPU_CTX_FPU_XSAVE_MASK(%rdi), %eax
165 movl FPU_CTX_FPU_XSAVE_MASK+4(%rdi), %edx
166 movq FPU_CTX_FPU_REGS(%rdi), %rsi /* fpu_regs.kfpu_u.kfpu_xs ptr */
167 xsaveopt (%rsi)
168 btw $7, FXSAVE_STATE_FSW(%rsi) /* Test saved ES bit */
169 jnc 0f /* jump if ES = 0 */
170 fnclex /* clear pending x87 exceptions */
171 0: ffree %st(7) /* clear tag bit to remove possible stack overflow */
172 fildl .fpzero_const(%rip) /* dummy load changes all excp. pointers */
173 STTS(%rsi) /* trap on next fpu touch */
174 1: ret
175 SET_SIZE(xsaveopt_excp_clr_ctxt)
176
177 .align 8
178 .fpzero_const:
179 .4byte 0x0
180 .4byte 0x0
181
182
183 ENTRY_NP(fpxsave)
184 CLTS
185 fxsaveq (%rdi)
186 fninit /* clear exceptions, init x87 tags */
187 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
188 ret
189 SET_SIZE(fpxsave)
190
191 ENTRY_NP(xsave)
192 CLTS
193 movl %esi, %eax /* bv mask */
194 movq %rsi, %rdx
195 shrq $32, %rdx
196 xsave (%rdi)
197
198 fninit /* clear exceptions, init x87 tags */
199 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
200 ret
201 SET_SIZE(xsave)
202
203 ENTRY_NP(xsaveopt)
204 CLTS
205 movl %esi, %eax /* bv mask */
206 movq %rsi, %rdx
207 shrq $32, %rdx
208 xsaveopt (%rdi)
209
210 fninit /* clear exceptions, init x87 tags */
211 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
212 ret
213 SET_SIZE(xsaveopt)
214
215 /*
216 * These functions are used when restoring the FPU as part of the epilogue of a
217 * context switch.
218 */
219
220 ENTRY(fpxrestore_ctxt)
221 cmpl $_CONST(FPU_EN|FPU_VALID), FPU_CTX_FPU_FLAGS(%rdi)
222 jne 1f
223 movl $_CONST(FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
224 movq FPU_CTX_FPU_REGS(%rdi), %rdi /* fpu_regs.kfpu_u.kfpu_fx ptr */
225 CLTS
226 fxrstorq (%rdi)
227 1:
228 ret
229 SET_SIZE(fpxrestore_ctxt)
230
231 ENTRY(xrestore_ctxt)
232 cmpl $_CONST(FPU_EN|FPU_VALID), FPU_CTX_FPU_FLAGS(%rdi)
233 jne 1f
234 movl $_CONST(FPU_EN), FPU_CTX_FPU_FLAGS(%rdi)
235 movl FPU_CTX_FPU_XSAVE_MASK(%rdi), %eax /* xsave flags in EDX:EAX */
236 movl FPU_CTX_FPU_XSAVE_MASK+4(%rdi), %edx
237 movq FPU_CTX_FPU_REGS(%rdi), %rdi /* fpu_regs.kfpu_u.kfpu_xs ptr */
238 CLTS
239 xrstor (%rdi)
240 1:
241 ret
242 SET_SIZE(xrestore_ctxt)
243
244
245 ENTRY_NP(fpxrestore)
246 CLTS
247 fxrstorq (%rdi)
248 ret
249 SET_SIZE(fpxrestore)
250
251 ENTRY_NP(xrestore)
252 CLTS
253 movl %esi, %eax /* bv mask */
254 movq %rsi, %rdx
255 shrq $32, %rdx
256 xrstor (%rdi)
257 ret
258 SET_SIZE(xrestore)
259
260 /*
261 * Disable the floating point unit.
262 */
263
264 ENTRY_NP(fpdisable)
265 STTS(%rdi) /* set TS bit in %cr0 (disable FPU) */
266 ret
267 SET_SIZE(fpdisable)
268
269 /*
270 * Initialize the fpu hardware.
271 */
272
273 ENTRY_NP(fpinit)
274 CLTS
275 cmpl $FP_XSAVE, fp_save_mech
276 je 1f
277
278 /* fxsave */
279 leaq sse_initial(%rip), %rax
280 fxrstorq (%rax) /* load clean initial state */
281 ret
282
283 1: /* xsave */
284 leaq avx_initial(%rip), %rcx
285 xorl %edx, %edx
286 movl $XFEATURE_AVX, %eax
287 bt $X86FSET_AVX, x86_featureset
288 cmovael %edx, %eax
289 orl $(XFEATURE_LEGACY_FP | XFEATURE_SSE), %eax
290 xrstor (%rcx)
291 ret
292 SET_SIZE(fpinit)
293
294 /*
295 * Clears FPU exception state.
296 * Returns the FP status word.
297 */
298
299 ENTRY_NP(fperr_reset)
300 CLTS
301 xorl %eax, %eax
302 fnstsw %ax
303 fnclex
304 ret
305 SET_SIZE(fperr_reset)
306
307 ENTRY_NP(fpxerr_reset)
308 pushq %rbp
309 movq %rsp, %rbp
310 subq $0x10, %rsp /* make some temporary space */
311 CLTS
312 stmxcsr (%rsp)
313 movl (%rsp), %eax
314 andl $_BITNOT(SSE_MXCSR_EFLAGS), (%rsp)
315 ldmxcsr (%rsp) /* clear processor exceptions */
316 leave
317 ret
318 SET_SIZE(fpxerr_reset)
319
320 ENTRY_NP(fpgetcwsw)
321 pushq %rbp
322 movq %rsp, %rbp
323 subq $0x10, %rsp /* make some temporary space */
324 CLTS
325 fnstsw (%rsp) /* store the status word */
326 fnstcw 2(%rsp) /* store the control word */
327 movl (%rsp), %eax /* put both in %eax */
328 leave
329 ret
330 SET_SIZE(fpgetcwsw)
331
332 /*
333 * Returns the MXCSR register.
334 */
335
336 ENTRY_NP(fpgetmxcsr)
337 pushq %rbp
338 movq %rsp, %rbp
339 subq $0x10, %rsp /* make some temporary space */
340 CLTS
341 stmxcsr (%rsp)
342 movl (%rsp), %eax
343 leave
344 ret
345 SET_SIZE(fpgetmxcsr)
346
|