2 #define FENV_PRIVATE_H 1
5 #include <fpu_control.h>
8 # define math_opt_barrier(x) \
10 if (sizeof (x) <= sizeof (double)) \
11 __asm ("" : "=x" (__x) : "0" (x)); \
13 __asm ("" : "=t" (__x) : "0" (x)); \
15 # define math_force_eval(x) \
17 if (sizeof (x) <= sizeof (double)) \
18 __asm __volatile ("" : : "x" (x)); \
20 __asm __volatile ("" : : "f" (x)); \
23 # define math_opt_barrier(x) \
24 ({ __typeof (x) __x; \
25 __asm ("" : "=t" (__x) : "0" (x)); \
27 # define math_force_eval(x) \
29 __typeof (x) __x = (x); \
30 if (sizeof (x) <= sizeof (double)) \
31 __asm __volatile ("" : : "m" (__x)); \
33 __asm __volatile ("" : : "f" (__x)); \
37 /* This file is used by both the 32- and 64-bit ports. The 64-bit port
38 has a field in the fenv_t for the mxcsr; the 32-bit port does not.
39 Instead, we (ab)use the only 32-bit field extant in the struct. */
41 # define __mxcsr __eip
45 /* All of these functions are private to libm, and are all used in pairs
46 to save+change the fp state and restore the original state. Thus we
47 need not care for both the 387 and the sse unit, only the one we're
50 #if defined __AVX__ || defined SSE2AVX
51 # define STMXCSR "vstmxcsr"
52 # define LDMXCSR "vldmxcsr"
54 # define STMXCSR "stmxcsr"
55 # define LDMXCSR "ldmxcsr"
58 static __always_inline
void
59 libc_feholdexcept_sse (fenv_t
*e
)
62 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
64 mxcsr
= (mxcsr
| 0x1f80) & ~0x3f;
65 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
68 static __always_inline
void
69 libc_feholdexcept_387 (fenv_t
*e
)
71 /* Recall that fnstenv has a side-effect of masking exceptions.
72 Clobber all of the fp registers so that the TOS field is 0. */
73 asm volatile ("fnstenv %0; fnclex"
75 : : "st", "st(1)", "st(2)", "st(3)",
76 "st(4)", "st(5)", "st(6)", "st(7)");
79 static __always_inline
void
80 libc_feholdexcept_setround_sse (fenv_t
*e
, int r
)
83 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
85 mxcsr
= ((mxcsr
| 0x1f80) & ~0x603f) | (r
<< 3);
86 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
89 /* Set both rounding mode and precision. A convenience function for use
90 by libc_feholdexcept_setround and libc_feholdexcept_setround_53bit. */
91 static __always_inline
void
92 libc_feholdexcept_setround_387_prec (fenv_t
*e
, int r
)
94 libc_feholdexcept_387 (e
);
96 fpu_control_t cw
= e
->__control_word
;
97 cw
&= ~(_FPU_RC_ZERO
| _FPU_EXTENDED
);
102 static __always_inline
void
103 libc_feholdexcept_setround_387 (fenv_t
*e
, int r
)
105 libc_feholdexcept_setround_387_prec (e
, r
| _FPU_EXTENDED
);
108 static __always_inline
void
109 libc_feholdexcept_setround_387_53bit (fenv_t
*e
, int r
)
111 libc_feholdexcept_setround_387_prec (e
, r
| _FPU_DOUBLE
);
114 static __always_inline
int
115 libc_fetestexcept_sse (int e
)
118 asm volatile (STMXCSR
" %0" : "=m" (*&mxcsr
));
119 return mxcsr
& e
& FE_ALL_EXCEPT
;
122 static __always_inline
int
123 libc_fetestexcept_387 (int ex
)
126 asm volatile ("fnstsw %0" : "=a" (temp
));
127 return temp
& ex
& FE_ALL_EXCEPT
;
130 static __always_inline
void
131 libc_fesetenv_sse (fenv_t
*e
)
133 asm volatile (LDMXCSR
" %0" : : "m" (e
->__mxcsr
));
136 static __always_inline
void
137 libc_fesetenv_387 (fenv_t
*e
)
139 /* Clobber all fp registers so that the TOS value we saved earlier is
140 compatible with the current state of the compiler. */
141 asm volatile ("fldenv %0"
143 : "st", "st(1)", "st(2)", "st(3)",
144 "st(4)", "st(5)", "st(6)", "st(7)");
147 static __always_inline
int
148 libc_feupdateenv_test_sse (fenv_t
*e
, int ex
)
150 unsigned int mxcsr
, old_mxcsr
, cur_ex
;
151 asm volatile (STMXCSR
" %0" : "=m" (*&mxcsr
));
152 cur_ex
= mxcsr
& FE_ALL_EXCEPT
;
154 /* Merge current exceptions with the old environment. */
155 old_mxcsr
= e
->__mxcsr
;
156 mxcsr
= old_mxcsr
| cur_ex
;
157 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
159 /* Raise SIGFPE for any new exceptions since the hold. Expect that
160 the normal environment has all exceptions masked. */
161 if (__builtin_expect ((old_mxcsr
>> 7) & cur_ex
, 0))
162 __feraiseexcept (cur_ex
);
164 /* Test for exceptions raised since the hold. */
168 static __always_inline
int
169 libc_feupdateenv_test_387 (fenv_t
*e
, int ex
)
173 /* Save current exceptions. */
174 asm volatile ("fnstsw %0" : "=a" (cur_ex
));
175 cur_ex
&= FE_ALL_EXCEPT
;
177 /* Reload original environment. */
178 libc_fesetenv_387 (e
);
180 /* Merge current exceptions. */
181 __feraiseexcept (cur_ex
);
183 /* Test for exceptions raised since the hold. */
187 static __always_inline
void
188 libc_feupdateenv_sse (fenv_t
*e
)
190 libc_feupdateenv_test_sse (e
, 0);
193 static __always_inline
void
194 libc_feupdateenv_387 (fenv_t
*e
)
196 libc_feupdateenv_test_387 (e
, 0);
199 static __always_inline
void
200 libc_feholdsetround_sse (fenv_t
*e
, int r
)
203 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
205 mxcsr
= (mxcsr
& ~0x6000) | (r
<< 3);
206 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
209 static __always_inline
void
210 libc_feholdsetround_387_prec (fenv_t
*e
, int r
)
215 e
->__control_word
= cw
;
216 cw
&= ~(_FPU_RC_ZERO
| _FPU_EXTENDED
);
221 static __always_inline
void
222 libc_feholdsetround_387 (fenv_t
*e
, int r
)
224 libc_feholdsetround_387_prec (e
, r
| _FPU_EXTENDED
);
227 static __always_inline
void
228 libc_feholdsetround_387_53bit (fenv_t
*e
, int r
)
230 libc_feholdsetround_387_prec (e
, r
| _FPU_DOUBLE
);
233 static __always_inline
void
234 libc_feresetround_sse (fenv_t
*e
)
237 asm (STMXCSR
" %0" : "=m" (*&mxcsr
));
238 mxcsr
= (mxcsr
& ~0x6000) | (e
->__mxcsr
& 0x6000);
239 asm volatile (LDMXCSR
" %0" : : "m" (*&mxcsr
));
242 static __always_inline
void
243 libc_feresetround_387 (fenv_t
*e
)
245 _FPU_SETCW (e
->__control_word
);
249 # define libc_feholdexceptf libc_feholdexcept_sse
250 # define libc_feholdexcept_setroundf libc_feholdexcept_setround_sse
251 # define libc_fetestexceptf libc_fetestexcept_sse
252 # define libc_fesetenvf libc_fesetenv_sse
253 # define libc_feupdateenv_testf libc_feupdateenv_test_sse
254 # define libc_feupdateenvf libc_feupdateenv_sse
255 # define libc_feholdsetroundf libc_feholdsetround_sse
256 # define libc_feresetroundf libc_feresetround_sse
258 # define libc_feholdexceptf libc_feholdexcept_387
259 # define libc_feholdexcept_setroundf libc_feholdexcept_setround_387
260 # define libc_fetestexceptf libc_fetestexcept_387
261 # define libc_fesetenvf libc_fesetenv_387
262 # define libc_feupdateenv_testf libc_feupdateenv_test_387
263 # define libc_feupdateenvf libc_feupdateenv_387
264 # define libc_feholdsetroundf libc_feholdsetround_387
265 # define libc_feresetroundf libc_feresetround_387
266 #endif /* __SSE_MATH__ */
269 # define libc_feholdexcept libc_feholdexcept_sse
270 # define libc_feholdexcept_setround libc_feholdexcept_setround_sse
271 # define libc_fetestexcept libc_fetestexcept_sse
272 # define libc_fesetenv libc_fesetenv_sse
273 # define libc_feupdateenv_test libc_feupdateenv_test_sse
274 # define libc_feupdateenv libc_feupdateenv_sse
275 # define libc_feholdsetround libc_feholdsetround_sse
276 # define libc_feresetround libc_feresetround_sse
278 # define libc_feholdexcept libc_feholdexcept_387
279 # define libc_feholdexcept_setround libc_feholdexcept_setround_387
280 # define libc_fetestexcept libc_fetestexcept_387
281 # define libc_fesetenv libc_fesetenv_387
282 # define libc_feupdateenv_test libc_feupdateenv_test_387
283 # define libc_feupdateenv libc_feupdateenv_387
284 # define libc_feholdsetround libc_feholdsetround_387
285 # define libc_feresetround libc_feresetround_387
286 #endif /* __SSE2_MATH__ */
288 #define libc_feholdexceptl libc_feholdexcept_387
289 #define libc_feholdexcept_setroundl libc_feholdexcept_setround_387
290 #define libc_fetestexceptl libc_fetestexcept_387
291 #define libc_fesetenvl libc_fesetenv_387
292 #define libc_feupdateenv_testl libc_feupdateenv_test_387
293 #define libc_feupdateenvl libc_feupdateenv_387
294 #define libc_feholdsetroundl libc_feholdsetround_387
295 #define libc_feresetroundl libc_feresetround_387
297 #ifndef __SSE2_MATH__
298 # define libc_feholdexcept_setround_53bit libc_feholdexcept_setround_387_53bit
299 # define libc_feholdsetround_53bit libc_feholdsetround_387_53bit
304 #endif /* FENV_PRIVATE_H */