1 #ifndef X86_FENV_PRIVATE_H
2 #define X86_FENV_PRIVATE_H 1
4 #include <bits/floatn.h>
6 #include <fpu_control.h>
8 /* This file is used by both the 32- and 64-bit ports. The 64-bit port
9 has a field in the fenv_t for the mxcsr; the 32-bit port does not.
10 Instead, we (ab)use the only 32-bit field extant in the struct. */
12 # define __mxcsr __eip
16 /* All of these functions are private to libm, and are all used in pairs
17 to save+change the fp state and restore the original state. Thus we
18 need not care for both the 387 and the sse unit, only the one we're
21 #if defined __AVX__ || defined SSE2AVX
22 # define STMXCSR "vstmxcsr"
23 # define LDMXCSR "vldmxcsr"
25 # define STMXCSR "stmxcsr"
26 # define LDMXCSR "ldmxcsr"
29 static __always_inline
void
30 libc_feholdexcept_sse (fenv_t
*e
)
33 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
35 mxcsr
= (mxcsr
| 0x1f80) & ~0x3f;
36 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
39 static __always_inline
void
40 libc_feholdexcept_387 (fenv_t
*e
)
42 /* Recall that fnstenv has a side-effect of masking exceptions.
43 Clobber all of the fp registers so that the TOS field is 0. */
44 asm volatile ("fnstenv %0; fnclex"
46 : : "st", "st(1)", "st(2)", "st(3)",
47 "st(4)", "st(5)", "st(6)", "st(7)");
50 static __always_inline
void
51 libc_fesetround_sse (int r
)
54 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
55 mxcsr
= (mxcsr
& ~0x6000) | (r
<< 3);
56 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
59 static __always_inline
void
60 libc_fesetround_387 (int r
)
64 cw
= (cw
& ~0xc00) | r
;
68 static __always_inline
void
69 libc_feholdexcept_setround_sse (fenv_t
*e
, int r
)
72 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
74 mxcsr
= ((mxcsr
| 0x1f80) & ~0x603f) | (r
<< 3);
75 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
78 /* Set both rounding mode and precision. A convenience function for use
79 by libc_feholdexcept_setround and libc_feholdexcept_setround_53bit. */
80 static __always_inline
void
81 libc_feholdexcept_setround_387_prec (fenv_t
*e
, int r
)
83 libc_feholdexcept_387 (e
);
85 fpu_control_t cw
= e
->__control_word
;
86 cw
&= ~(_FPU_RC_ZERO
| _FPU_EXTENDED
);
91 static __always_inline
void
92 libc_feholdexcept_setround_387 (fenv_t
*e
, int r
)
94 libc_feholdexcept_setround_387_prec (e
, r
| _FPU_EXTENDED
);
97 static __always_inline
void
98 libc_feholdexcept_setround_387_53bit (fenv_t
*e
, int r
)
100 libc_feholdexcept_setround_387_prec (e
, r
| _FPU_DOUBLE
);
103 static __always_inline
int
104 libc_fetestexcept_sse (int e
)
107 asm volatile (STMXCSR
" %0" : "=m" (*&mxcsr
));
108 return mxcsr
& e
& FE_ALL_EXCEPT
;
111 static __always_inline
int
112 libc_fetestexcept_387 (int ex
)
115 asm volatile ("fnstsw %0" : "=a" (temp
));
116 return temp
& ex
& FE_ALL_EXCEPT
;
119 static __always_inline
void
120 libc_fesetenv_sse (fenv_t
*e
)
122 asm volatile (LDMXCSR
" %0" : : "m" (e
->__mxcsr
));
125 static __always_inline
void
126 libc_fesetenv_387 (fenv_t
*e
)
128 /* Clobber all fp registers so that the TOS value we saved earlier is
129 compatible with the current state of the compiler. */
130 asm volatile ("fldenv %0"
132 : "st", "st(1)", "st(2)", "st(3)",
133 "st(4)", "st(5)", "st(6)", "st(7)");
136 static __always_inline
int
137 libc_feupdateenv_test_sse (fenv_t
*e
, int ex
)
139 unsigned int mxcsr
, old_mxcsr
, cur_ex
;
140 asm volatile (STMXCSR
" %0" : "=m" (*&mxcsr
));
141 cur_ex
= mxcsr
& FE_ALL_EXCEPT
;
143 /* Merge current exceptions with the old environment. */
144 old_mxcsr
= e
->__mxcsr
;
145 mxcsr
= old_mxcsr
| cur_ex
;
146 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
148 /* Raise SIGFPE for any new exceptions since the hold. Expect that
149 the normal environment has all exceptions masked. */
150 if (__glibc_unlikely (~(old_mxcsr
>> 7) & cur_ex
))
151 __feraiseexcept (cur_ex
);
153 /* Test for exceptions raised since the hold. */
157 static __always_inline
int
158 libc_feupdateenv_test_387 (fenv_t
*e
, int ex
)
162 /* Save current exceptions. */
163 asm volatile ("fnstsw %0" : "=a" (cur_ex
));
164 cur_ex
&= FE_ALL_EXCEPT
;
166 /* Reload original environment. */
167 libc_fesetenv_387 (e
);
169 /* Merge current exceptions. */
170 __feraiseexcept (cur_ex
);
172 /* Test for exceptions raised since the hold. */
176 static __always_inline
void
177 libc_feupdateenv_sse (fenv_t
*e
)
179 libc_feupdateenv_test_sse (e
, 0);
182 static __always_inline
void
183 libc_feupdateenv_387 (fenv_t
*e
)
185 libc_feupdateenv_test_387 (e
, 0);
188 static __always_inline
void
189 libc_feholdsetround_sse (fenv_t
*e
, int r
)
192 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
194 mxcsr
= (mxcsr
& ~0x6000) | (r
<< 3);
195 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
198 static __always_inline
void
199 libc_feholdsetround_387_prec (fenv_t
*e
, int r
)
204 e
->__control_word
= cw
;
205 cw
&= ~(_FPU_RC_ZERO
| _FPU_EXTENDED
);
210 static __always_inline
void
211 libc_feholdsetround_387 (fenv_t
*e
, int r
)
213 libc_feholdsetround_387_prec (e
, r
| _FPU_EXTENDED
);
216 static __always_inline
void
217 libc_feholdsetround_387_53bit (fenv_t
*e
, int r
)
219 libc_feholdsetround_387_prec (e
, r
| _FPU_DOUBLE
);
222 static __always_inline
void
223 libc_feresetround_sse (fenv_t
*e
)
226 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
227 mxcsr
= (mxcsr
& ~0x6000) | (e
->__mxcsr
& 0x6000);
228 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
231 static __always_inline
void
232 libc_feresetround_387 (fenv_t
*e
)
234 _FPU_SETCW (e
->__control_word
);
238 # define libc_feholdexceptf libc_feholdexcept_sse
239 # define libc_fesetroundf libc_fesetround_sse
240 # define libc_feholdexcept_setroundf libc_feholdexcept_setround_sse
241 # define libc_fetestexceptf libc_fetestexcept_sse
242 # define libc_fesetenvf libc_fesetenv_sse
243 # define libc_feupdateenv_testf libc_feupdateenv_test_sse
244 # define libc_feupdateenvf libc_feupdateenv_sse
245 # define libc_feholdsetroundf libc_feholdsetround_sse
246 # define libc_feresetroundf libc_feresetround_sse
248 # define libc_feholdexceptf libc_feholdexcept_387
249 # define libc_fesetroundf libc_fesetround_387
250 # define libc_feholdexcept_setroundf libc_feholdexcept_setround_387
251 # define libc_fetestexceptf libc_fetestexcept_387
252 # define libc_fesetenvf libc_fesetenv_387
253 # define libc_feupdateenv_testf libc_feupdateenv_test_387
254 # define libc_feupdateenvf libc_feupdateenv_387
255 # define libc_feholdsetroundf libc_feholdsetround_387
256 # define libc_feresetroundf libc_feresetround_387
257 #endif /* __SSE_MATH__ */
260 # define libc_feholdexcept libc_feholdexcept_sse
261 # define libc_fesetround libc_fesetround_sse
262 # define libc_feholdexcept_setround libc_feholdexcept_setround_sse
263 # define libc_fetestexcept libc_fetestexcept_sse
264 # define libc_fesetenv libc_fesetenv_sse
265 # define libc_feupdateenv_test libc_feupdateenv_test_sse
266 # define libc_feupdateenv libc_feupdateenv_sse
267 # define libc_feholdsetround libc_feholdsetround_sse
268 # define libc_feresetround libc_feresetround_sse
270 # define libc_feholdexcept libc_feholdexcept_387
271 # define libc_fesetround libc_fesetround_387
272 # define libc_feholdexcept_setround libc_feholdexcept_setround_387
273 # define libc_fetestexcept libc_fetestexcept_387
274 # define libc_fesetenv libc_fesetenv_387
275 # define libc_feupdateenv_test libc_feupdateenv_test_387
276 # define libc_feupdateenv libc_feupdateenv_387
277 # define libc_feholdsetround libc_feholdsetround_387
278 # define libc_feresetround libc_feresetround_387
279 #endif /* __SSE2_MATH__ */
281 #define libc_feholdexceptl libc_feholdexcept_387
282 #define libc_fesetroundl libc_fesetround_387
283 #define libc_feholdexcept_setroundl libc_feholdexcept_setround_387
284 #define libc_fetestexceptl libc_fetestexcept_387
285 #define libc_fesetenvl libc_fesetenv_387
286 #define libc_feupdateenv_testl libc_feupdateenv_test_387
287 #define libc_feupdateenvl libc_feupdateenv_387
288 #define libc_feholdsetroundl libc_feholdsetround_387
289 #define libc_feresetroundl libc_feresetround_387
291 #ifndef __SSE2_MATH__
292 # define libc_feholdexcept_setround_53bit libc_feholdexcept_setround_387_53bit
293 # define libc_feholdsetround_53bit libc_feholdsetround_387_53bit
297 /* The SSE rounding mode is used by soft-fp (libgcc and glibc) on
298 x86_64, so that must be set for float128 computations. */
299 # define SET_RESTORE_ROUNDF128(RM) \
300 SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_sse, libc_feresetround_sse)
301 # define libc_feholdexcept_setroundf128 libc_feholdexcept_setround_sse
302 # define libc_feupdateenv_testf128 libc_feupdateenv_test_sse
304 /* The 387 rounding mode is used by soft-fp for 32-bit, but whether
305 387 or SSE exceptions are used depends on whether libgcc was built
306 for SSE math, which is not known when glibc is being built. */
307 # define libc_feholdexcept_setroundf128 default_libc_feholdexcept_setround
308 # define libc_feupdateenv_testf128 default_libc_feupdateenv_test
311 /* We have support for rounding mode context. */
312 #define HAVE_RM_CTX 1
314 static __always_inline
void
315 libc_feholdexcept_setround_sse_ctx (struct rm_ctx
*ctx
, int r
)
317 unsigned int mxcsr
, new_mxcsr
;
318 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
319 new_mxcsr
= ((mxcsr
| 0x1f80) & ~0x603f) | (r
<< 3);
321 ctx
->env
.__mxcsr
= mxcsr
;
322 if (__glibc_unlikely (mxcsr
!= new_mxcsr
))
324 asm volatile (LDMXCSR
" %0" : : "m" (*&new_mxcsr
));
325 ctx
->updated_status
= true;
328 ctx
->updated_status
= false;
331 /* Unconditional since we want to overwrite any exceptions that occurred in the
332 context. This is also why all fehold* functions unconditionally write into
334 static __always_inline
void
335 libc_fesetenv_sse_ctx (struct rm_ctx
*ctx
)
337 libc_fesetenv_sse (&ctx
->env
);
340 static __always_inline
void
341 libc_feupdateenv_sse_ctx (struct rm_ctx
*ctx
)
343 if (__glibc_unlikely (ctx
->updated_status
))
344 libc_feupdateenv_test_sse (&ctx
->env
, 0);
347 static __always_inline
void
348 libc_feholdexcept_setround_387_prec_ctx (struct rm_ctx
*ctx
, int r
)
350 libc_feholdexcept_387 (&ctx
->env
);
352 fpu_control_t cw
= ctx
->env
.__control_word
;
353 fpu_control_t old_cw
= cw
;
354 cw
&= ~(_FPU_RC_ZERO
| _FPU_EXTENDED
);
357 if (__glibc_unlikely (old_cw
!= cw
))
360 ctx
->updated_status
= true;
363 ctx
->updated_status
= false;
366 static __always_inline
void
367 libc_feholdexcept_setround_387_ctx (struct rm_ctx
*ctx
, int r
)
369 libc_feholdexcept_setround_387_prec_ctx (ctx
, r
| _FPU_EXTENDED
);
372 static __always_inline
void
373 libc_feholdexcept_setround_387_53bit_ctx (struct rm_ctx
*ctx
, int r
)
375 libc_feholdexcept_setround_387_prec_ctx (ctx
, r
| _FPU_DOUBLE
);
378 static __always_inline
void
379 libc_feholdsetround_387_prec_ctx (struct rm_ctx
*ctx
, int r
)
381 fpu_control_t cw
, new_cw
;
385 new_cw
&= ~(_FPU_RC_ZERO
| _FPU_EXTENDED
);
388 ctx
->env
.__control_word
= cw
;
389 if (__glibc_unlikely (new_cw
!= cw
))
392 ctx
->updated_status
= true;
395 ctx
->updated_status
= false;
398 static __always_inline
void
399 libc_feholdsetround_387_ctx (struct rm_ctx
*ctx
, int r
)
401 libc_feholdsetround_387_prec_ctx (ctx
, r
| _FPU_EXTENDED
);
404 static __always_inline
void
405 libc_feholdsetround_387_53bit_ctx (struct rm_ctx
*ctx
, int r
)
407 libc_feholdsetround_387_prec_ctx (ctx
, r
| _FPU_DOUBLE
);
410 static __always_inline
void
411 libc_feholdsetround_sse_ctx (struct rm_ctx
*ctx
, int r
)
413 unsigned int mxcsr
, new_mxcsr
;
415 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
416 new_mxcsr
= (mxcsr
& ~0x6000) | (r
<< 3);
418 ctx
->env
.__mxcsr
= mxcsr
;
419 if (__glibc_unlikely (new_mxcsr
!= mxcsr
))
421 asm volatile (LDMXCSR
" %0" : : "m" (*&new_mxcsr
));
422 ctx
->updated_status
= true;
425 ctx
->updated_status
= false;
428 static __always_inline
void
429 libc_feresetround_sse_ctx (struct rm_ctx
*ctx
)
431 if (__glibc_unlikely (ctx
->updated_status
))
432 libc_feresetround_sse (&ctx
->env
);
435 static __always_inline
void
436 libc_feresetround_387_ctx (struct rm_ctx
*ctx
)
438 if (__glibc_unlikely (ctx
->updated_status
))
439 _FPU_SETCW (ctx
->env
.__control_word
);
442 static __always_inline
void
443 libc_feupdateenv_387_ctx (struct rm_ctx
*ctx
)
445 if (__glibc_unlikely (ctx
->updated_status
))
446 libc_feupdateenv_test_387 (&ctx
->env
, 0);
450 # define libc_feholdexcept_setroundf_ctx libc_feholdexcept_setround_sse_ctx
451 # define libc_fesetenvf_ctx libc_fesetenv_sse_ctx
452 # define libc_feupdateenvf_ctx libc_feupdateenv_sse_ctx
453 # define libc_feholdsetroundf_ctx libc_feholdsetround_sse_ctx
454 # define libc_feresetroundf_ctx libc_feresetround_sse_ctx
456 # define libc_feholdexcept_setroundf_ctx libc_feholdexcept_setround_387_ctx
457 # define libc_feupdateenvf_ctx libc_feupdateenv_387_ctx
458 # define libc_feholdsetroundf_ctx libc_feholdsetround_387_ctx
459 # define libc_feresetroundf_ctx libc_feresetround_387_ctx
460 #endif /* __SSE_MATH__ */
463 # if defined (__x86_64__) || !defined (MATH_SET_BOTH_ROUNDING_MODES)
464 # define libc_feholdexcept_setround_ctx libc_feholdexcept_setround_sse_ctx
465 # define libc_fesetenv_ctx libc_fesetenv_sse_ctx
466 # define libc_feupdateenv_ctx libc_feupdateenv_sse_ctx
467 # define libc_feholdsetround_ctx libc_feholdsetround_sse_ctx
468 # define libc_feresetround_ctx libc_feresetround_sse_ctx
470 # define libc_feholdexcept_setround_ctx default_libc_feholdexcept_setround_ctx
471 # define libc_fesetenv_ctx default_libc_fesetenv_ctx
472 # define libc_feupdateenv_ctx default_libc_feupdateenv_ctx
473 # define libc_feholdsetround_ctx default_libc_feholdsetround_ctx
474 # define libc_feresetround_ctx default_libc_feresetround_ctx
477 # define libc_feholdexcept_setround_ctx libc_feholdexcept_setround_387_ctx
478 # define libc_feupdateenv_ctx libc_feupdateenv_387_ctx
479 # define libc_feholdsetround_ctx libc_feholdsetround_387_ctx
480 # define libc_feresetround_ctx libc_feresetround_387_ctx
481 #endif /* __SSE2_MATH__ */
483 #define libc_feholdexcept_setroundl_ctx libc_feholdexcept_setround_387_ctx
484 #define libc_feupdateenvl_ctx libc_feupdateenv_387_ctx
485 #define libc_feholdsetroundl_ctx libc_feholdsetround_387_ctx
486 #define libc_feresetroundl_ctx libc_feresetround_387_ctx
488 #ifndef __SSE2_MATH__
489 # define libc_feholdsetround_53bit_ctx libc_feholdsetround_387_53bit_ctx
490 # define libc_feresetround_53bit_ctx libc_feresetround_387_ctx
495 #include_next <fenv_private.h>
497 #endif /* X86_FENV_PRIVATE_H */