2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2014,2015,2016,2017,2018 by the GROMACS development team.
5 * Copyright (c) 2019,2020, by the GROMACS development team, led by
6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 * and including many others, as listed in the AUTHORS file in the
8 * top-level source directory and at http://www.gromacs.org.
10 * GROMACS is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2.1
13 * of the License, or (at your option) any later version.
15 * GROMACS is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with GROMACS; if not, see
22 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 * If you want to redistribute modifications to GROMACS, please
26 * consider that scientific software is very special. Version
27 * control is crucial - bugs must be traceable. We will be happy to
28 * consider code for inclusion in the official distribution, but
29 * derived work must not be called official GROMACS. Details are found
30 * in the README & COPYING files - if they are missing, get the
31 * official version at http://www.gromacs.org.
33 * To help us fund GROMACS development, we humbly ask that you cite
34 * the research papers on the package. Check out http://www.gromacs.org.
37 #ifndef GMX_SIMD_IMPL_X86_AVX_512_SIMD_FLOAT_H
38 #define GMX_SIMD_IMPL_X86_AVX_512_SIMD_FLOAT_H
45 #include <immintrin.h>
47 #include "gromacs/math/utilities.h"
48 #include "gromacs/utility/real.h"
50 #include "impl_x86_avx_512_general.h"
60 SimdFloat(float f
) : simdInternal_(_mm512_set1_ps(f
)) {}
62 // Internal utility constructor to simplify return statements
63 SimdFloat(__m512 simd
) : simdInternal_(simd
) {}
73 SimdFInt32(std::int32_t i
) : simdInternal_(_mm512_set1_epi32(i
)) {}
75 // Internal utility constructor to simplify return statements
76 SimdFInt32(__m512i simd
) : simdInternal_(simd
) {}
78 __m512i simdInternal_
;
86 // Internal utility constructor to simplify return statements
87 SimdFBool(__mmask16 simd
) : simdInternal_(simd
) {}
89 __mmask16 simdInternal_
;
97 // Internal utility constructor to simplify return statements
98 SimdFIBool(__mmask16 simd
) : simdInternal_(simd
) {}
100 __mmask16 simdInternal_
;
103 static inline SimdFloat gmx_simdcall
simdLoad(const float* m
, SimdFloatTag
= {})
105 assert(std::size_t(m
) % 64 == 0);
106 return { _mm512_load_ps(m
) };
109 static inline void gmx_simdcall
store(float* m
, SimdFloat a
)
111 assert(std::size_t(m
) % 64 == 0);
112 _mm512_store_ps(m
, a
.simdInternal_
);
115 static inline SimdFloat gmx_simdcall
simdLoadU(const float* m
, SimdFloatTag
= {})
117 return { _mm512_loadu_ps(m
) };
120 static inline void gmx_simdcall
storeU(float* m
, SimdFloat a
)
122 _mm512_storeu_ps(m
, a
.simdInternal_
);
125 static inline SimdFloat gmx_simdcall
setZeroF()
127 return { _mm512_setzero_ps() };
130 static inline SimdFInt32 gmx_simdcall
simdLoad(const std::int32_t* m
, SimdFInt32Tag
)
132 assert(std::size_t(m
) % 64 == 0);
133 return { _mm512_load_si512(m
) };
136 static inline void gmx_simdcall
store(std::int32_t* m
, SimdFInt32 a
)
138 assert(std::size_t(m
) % 64 == 0);
139 _mm512_store_si512(m
, a
.simdInternal_
);
142 static inline SimdFInt32 gmx_simdcall
simdLoadU(const std::int32_t* m
, SimdFInt32Tag
)
144 return { _mm512_loadu_si512(m
) };
147 static inline void gmx_simdcall
storeU(std::int32_t* m
, SimdFInt32 a
)
149 _mm512_storeu_si512(m
, a
.simdInternal_
);
152 static inline SimdFInt32 gmx_simdcall
setZeroFI()
154 return { _mm512_setzero_si512() };
158 static inline SimdFloat gmx_simdcall
operator&(SimdFloat a
, SimdFloat b
)
160 return { _mm512_castsi512_ps(_mm512_and_epi32(_mm512_castps_si512(a
.simdInternal_
),
161 _mm512_castps_si512(b
.simdInternal_
))) };
164 static inline SimdFloat gmx_simdcall
andNot(SimdFloat a
, SimdFloat b
)
166 return { _mm512_castsi512_ps(_mm512_andnot_epi32(_mm512_castps_si512(a
.simdInternal_
),
167 _mm512_castps_si512(b
.simdInternal_
))) };
170 static inline SimdFloat gmx_simdcall
operator|(SimdFloat a
, SimdFloat b
)
172 return { _mm512_castsi512_ps(_mm512_or_epi32(_mm512_castps_si512(a
.simdInternal_
),
173 _mm512_castps_si512(b
.simdInternal_
))) };
176 static inline SimdFloat gmx_simdcall
operator^(SimdFloat a
, SimdFloat b
)
178 return { _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a
.simdInternal_
),
179 _mm512_castps_si512(b
.simdInternal_
))) };
182 static inline SimdFloat gmx_simdcall
operator+(SimdFloat a
, SimdFloat b
)
184 return { _mm512_add_ps(a
.simdInternal_
, b
.simdInternal_
) };
187 static inline SimdFloat gmx_simdcall
operator-(SimdFloat a
, SimdFloat b
)
189 return { _mm512_sub_ps(a
.simdInternal_
, b
.simdInternal_
) };
192 static inline SimdFloat gmx_simdcall
operator-(SimdFloat x
)
194 return { _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(x
.simdInternal_
),
195 _mm512_castps_si512(_mm512_set1_ps(GMX_FLOAT_NEGZERO
)))) };
198 static inline SimdFloat gmx_simdcall
operator*(SimdFloat a
, SimdFloat b
)
200 return { _mm512_mul_ps(a
.simdInternal_
, b
.simdInternal_
) };
203 static inline SimdFloat gmx_simdcall
fma(SimdFloat a
, SimdFloat b
, SimdFloat c
)
205 return { _mm512_fmadd_ps(a
.simdInternal_
, b
.simdInternal_
, c
.simdInternal_
) };
208 static inline SimdFloat gmx_simdcall
fms(SimdFloat a
, SimdFloat b
, SimdFloat c
)
210 return { _mm512_fmsub_ps(a
.simdInternal_
, b
.simdInternal_
, c
.simdInternal_
) };
213 static inline SimdFloat gmx_simdcall
fnma(SimdFloat a
, SimdFloat b
, SimdFloat c
)
215 return { _mm512_fnmadd_ps(a
.simdInternal_
, b
.simdInternal_
, c
.simdInternal_
) };
218 static inline SimdFloat gmx_simdcall
fnms(SimdFloat a
, SimdFloat b
, SimdFloat c
)
220 return { _mm512_fnmsub_ps(a
.simdInternal_
, b
.simdInternal_
, c
.simdInternal_
) };
223 // Override for AVX-512-KNL
224 #if GMX_SIMD_X86_AVX_512
225 static inline SimdFloat gmx_simdcall
rsqrt(SimdFloat x
)
227 return { _mm512_rsqrt14_ps(x
.simdInternal_
) };
230 static inline SimdFloat gmx_simdcall
rcp(SimdFloat x
)
232 return { _mm512_rcp14_ps(x
.simdInternal_
) };
236 static inline SimdFloat gmx_simdcall
maskAdd(SimdFloat a
, SimdFloat b
, SimdFBool m
)
238 return { _mm512_mask_add_ps(a
.simdInternal_
, m
.simdInternal_
, a
.simdInternal_
, b
.simdInternal_
) };
241 static inline SimdFloat gmx_simdcall
maskzMul(SimdFloat a
, SimdFloat b
, SimdFBool m
)
243 return { _mm512_maskz_mul_ps(m
.simdInternal_
, a
.simdInternal_
, b
.simdInternal_
) };
246 static inline SimdFloat gmx_simdcall
maskzFma(SimdFloat a
, SimdFloat b
, SimdFloat c
, SimdFBool m
)
248 return { _mm512_maskz_fmadd_ps(m
.simdInternal_
, a
.simdInternal_
, b
.simdInternal_
, c
.simdInternal_
) };
251 // Override for AVX-512-KNL
252 #if GMX_SIMD_X86_AVX_512
253 static inline SimdFloat gmx_simdcall
maskzRsqrt(SimdFloat x
, SimdFBool m
)
255 return { _mm512_maskz_rsqrt14_ps(m
.simdInternal_
, x
.simdInternal_
) };
258 static inline SimdFloat gmx_simdcall
maskzRcp(SimdFloat x
, SimdFBool m
)
260 return { _mm512_maskz_rcp14_ps(m
.simdInternal_
, x
.simdInternal_
) };
264 static inline SimdFloat gmx_simdcall
abs(SimdFloat x
)
266 return { _mm512_castsi512_ps(_mm512_andnot_epi32(_mm512_castps_si512(_mm512_set1_ps(GMX_FLOAT_NEGZERO
)),
267 _mm512_castps_si512(x
.simdInternal_
))) };
270 static inline SimdFloat gmx_simdcall
max(SimdFloat a
, SimdFloat b
)
272 return { _mm512_max_ps(a
.simdInternal_
, b
.simdInternal_
) };
275 static inline SimdFloat gmx_simdcall
min(SimdFloat a
, SimdFloat b
)
277 return { _mm512_min_ps(a
.simdInternal_
, b
.simdInternal_
) };
280 static inline SimdFloat gmx_simdcall
round(SimdFloat x
)
282 return { _mm512_roundscale_ps(x
.simdInternal_
, 0) };
285 static inline SimdFloat gmx_simdcall
trunc(SimdFloat x
)
287 #if defined(__INTEL_COMPILER) || defined(__ECC)
288 return { _mm512_trunc_ps(x
.simdInternal_
) };
290 return { _mm512_cvtepi32_ps(_mm512_cvttps_epi32(x
.simdInternal_
)) };
294 template<MathOptimization opt
= MathOptimization::Safe
>
295 static inline SimdFloat gmx_simdcall
frexp(SimdFloat value
, SimdFInt32
* exponent
)
301 if (opt
== MathOptimization::Safe
)
303 // For the safe branch, we use the masked operations to only assign results if the
304 // input value was nonzero, and otherwise set exponent to 0, and the fraction to the input (+-0).
305 __mmask16 valueIsNonZero
=
306 _mm512_cmp_ps_mask(_mm512_setzero_ps(), value
.simdInternal_
, _CMP_NEQ_OQ
);
307 rExponent
= _mm512_mask_getexp_ps(_mm512_setzero_ps(), valueIsNonZero
, value
.simdInternal_
);
308 iExponent
= _mm512_cvtps_epi32(rExponent
);
309 iExponent
= _mm512_mask_add_epi32(iExponent
, valueIsNonZero
, iExponent
, _mm512_set1_epi32(1));
311 // Set result to input value when the latter is +-0
312 result
= _mm512_mask_getmant_ps(value
.simdInternal_
, valueIsNonZero
, value
.simdInternal_
,
313 _MM_MANT_NORM_p5_1
, _MM_MANT_SIGN_src
);
317 // For the fast branch, it's the user's responsibility to make sure never to call the
318 // function with input values of +-0.0
319 rExponent
= _mm512_getexp_ps(value
.simdInternal_
);
320 iExponent
= _mm512_cvtps_epi32(rExponent
);
321 iExponent
= _mm512_add_epi32(iExponent
, _mm512_set1_epi32(1));
323 result
= _mm512_getmant_ps(value
.simdInternal_
, _MM_MANT_NORM_p5_1
, _MM_MANT_SIGN_src
);
326 exponent
->simdInternal_
= iExponent
;
331 template<MathOptimization opt
= MathOptimization::Safe
>
332 static inline SimdFloat gmx_simdcall
ldexp(SimdFloat value
, SimdFInt32 exponent
)
334 const __m512i exponentBias
= _mm512_set1_epi32(127);
335 __m512i iExponent
= _mm512_add_epi32(exponent
.simdInternal_
, exponentBias
);
337 if (opt
== MathOptimization::Safe
)
339 // Make sure biased argument is not negative
340 iExponent
= _mm512_max_epi32(iExponent
, _mm512_setzero_epi32());
343 iExponent
= _mm512_slli_epi32(iExponent
, 23);
345 return { _mm512_mul_ps(value
.simdInternal_
, _mm512_castsi512_ps(iExponent
)) };
348 static inline float gmx_simdcall
reduce(SimdFloat a
)
350 __m512 x
= a
.simdInternal_
;
351 x
= _mm512_add_ps(x
, _mm512_shuffle_f32x4(x
, x
, 0xEE));
352 x
= _mm512_add_ps(x
, _mm512_shuffle_f32x4(x
, x
, 0x11));
353 x
= _mm512_add_ps(x
, _mm512_permute_ps(x
, 0xEE));
354 x
= _mm512_add_ps(x
, _mm512_permute_ps(x
, 0x11));
355 return *reinterpret_cast<float*>(&x
);
358 static inline SimdFBool gmx_simdcall
operator==(SimdFloat a
, SimdFloat b
)
360 return { _mm512_cmp_ps_mask(a
.simdInternal_
, b
.simdInternal_
, _CMP_EQ_OQ
) };
363 static inline SimdFBool gmx_simdcall
operator!=(SimdFloat a
, SimdFloat b
)
365 return { _mm512_cmp_ps_mask(a
.simdInternal_
, b
.simdInternal_
, _CMP_NEQ_OQ
) };
368 static inline SimdFBool gmx_simdcall
operator<(SimdFloat a
, SimdFloat b
)
370 return { _mm512_cmp_ps_mask(a
.simdInternal_
, b
.simdInternal_
, _CMP_LT_OQ
) };
373 static inline SimdFBool gmx_simdcall
operator<=(SimdFloat a
, SimdFloat b
)
375 return { _mm512_cmp_ps_mask(a
.simdInternal_
, b
.simdInternal_
, _CMP_LE_OQ
) };
378 static inline SimdFBool gmx_simdcall
testBits(SimdFloat a
)
380 return { _mm512_test_epi32_mask(_mm512_castps_si512(a
.simdInternal_
),
381 _mm512_castps_si512(a
.simdInternal_
)) };
384 static inline SimdFBool gmx_simdcall
operator&&(SimdFBool a
, SimdFBool b
)
386 return { _mm512_kand(a
.simdInternal_
, b
.simdInternal_
) };
389 static inline SimdFBool gmx_simdcall
operator||(SimdFBool a
, SimdFBool b
)
391 return { _mm512_kor(a
.simdInternal_
, b
.simdInternal_
) };
394 static inline bool gmx_simdcall
anyTrue(SimdFBool a
)
396 return (avx512Mask2Int(a
.simdInternal_
) != 0);
399 static inline SimdFloat gmx_simdcall
selectByMask(SimdFloat a
, SimdFBool m
)
401 return { _mm512_mask_mov_ps(_mm512_setzero_ps(), m
.simdInternal_
, a
.simdInternal_
) };
404 static inline SimdFloat gmx_simdcall
selectByNotMask(SimdFloat a
, SimdFBool m
)
406 return { _mm512_mask_mov_ps(a
.simdInternal_
, m
.simdInternal_
, _mm512_setzero_ps()) };
409 static inline SimdFloat gmx_simdcall
blend(SimdFloat a
, SimdFloat b
, SimdFBool sel
)
411 return { _mm512_mask_blend_ps(sel
.simdInternal_
, a
.simdInternal_
, b
.simdInternal_
) };
414 static inline SimdFloat gmx_simdcall
copysign(SimdFloat a
, SimdFloat b
)
416 return { _mm512_castsi512_ps(_mm512_ternarylogic_epi32(_mm512_castps_si512(a
.simdInternal_
),
417 _mm512_castps_si512(b
.simdInternal_
),
418 _mm512_set1_epi32(INT32_MIN
), 0xD8)) };
421 static inline SimdFInt32 gmx_simdcall
operator&(SimdFInt32 a
, SimdFInt32 b
)
423 return { _mm512_and_epi32(a
.simdInternal_
, b
.simdInternal_
) };
426 static inline SimdFInt32 gmx_simdcall
andNot(SimdFInt32 a
, SimdFInt32 b
)
428 return { _mm512_andnot_epi32(a
.simdInternal_
, b
.simdInternal_
) };
431 static inline SimdFInt32 gmx_simdcall
operator|(SimdFInt32 a
, SimdFInt32 b
)
433 return { _mm512_or_epi32(a
.simdInternal_
, b
.simdInternal_
) };
436 static inline SimdFInt32 gmx_simdcall
operator^(SimdFInt32 a
, SimdFInt32 b
)
438 return { _mm512_xor_epi32(a
.simdInternal_
, b
.simdInternal_
) };
441 static inline SimdFInt32 gmx_simdcall
operator+(SimdFInt32 a
, SimdFInt32 b
)
443 return { _mm512_add_epi32(a
.simdInternal_
, b
.simdInternal_
) };
446 static inline SimdFInt32 gmx_simdcall
operator-(SimdFInt32 a
, SimdFInt32 b
)
448 return { _mm512_sub_epi32(a
.simdInternal_
, b
.simdInternal_
) };
451 static inline SimdFInt32 gmx_simdcall
operator*(SimdFInt32 a
, SimdFInt32 b
)
453 return { _mm512_mullo_epi32(a
.simdInternal_
, b
.simdInternal_
) };
456 static inline SimdFIBool gmx_simdcall
operator==(SimdFInt32 a
, SimdFInt32 b
)
458 return { _mm512_cmp_epi32_mask(a
.simdInternal_
, b
.simdInternal_
, _MM_CMPINT_EQ
) };
461 static inline SimdFIBool gmx_simdcall
testBits(SimdFInt32 a
)
463 return { _mm512_test_epi32_mask(a
.simdInternal_
, a
.simdInternal_
) };
466 static inline SimdFIBool gmx_simdcall
operator<(SimdFInt32 a
, SimdFInt32 b
)
468 return { _mm512_cmp_epi32_mask(a
.simdInternal_
, b
.simdInternal_
, _MM_CMPINT_LT
) };
471 static inline SimdFIBool gmx_simdcall
operator&&(SimdFIBool a
, SimdFIBool b
)
473 return { _mm512_kand(a
.simdInternal_
, b
.simdInternal_
) };
476 static inline SimdFIBool gmx_simdcall
operator||(SimdFIBool a
, SimdFIBool b
)
478 return { _mm512_kor(a
.simdInternal_
, b
.simdInternal_
) };
481 static inline bool gmx_simdcall
anyTrue(SimdFIBool a
)
483 return (avx512Mask2Int(a
.simdInternal_
) != 0);
486 static inline SimdFInt32 gmx_simdcall
selectByMask(SimdFInt32 a
, SimdFIBool m
)
488 return { _mm512_mask_mov_epi32(_mm512_setzero_epi32(), m
.simdInternal_
, a
.simdInternal_
) };
491 static inline SimdFInt32 gmx_simdcall
selectByNotMask(SimdFInt32 a
, SimdFIBool m
)
493 return { _mm512_mask_mov_epi32(a
.simdInternal_
, m
.simdInternal_
, _mm512_setzero_epi32()) };
496 static inline SimdFInt32 gmx_simdcall
blend(SimdFInt32 a
, SimdFInt32 b
, SimdFIBool sel
)
498 return { _mm512_mask_blend_epi32(sel
.simdInternal_
, a
.simdInternal_
, b
.simdInternal_
) };
501 static inline SimdFInt32 gmx_simdcall
cvtR2I(SimdFloat a
)
503 return { _mm512_cvtps_epi32(a
.simdInternal_
) };
506 static inline SimdFInt32 gmx_simdcall
cvttR2I(SimdFloat a
)
508 return { _mm512_cvttps_epi32(a
.simdInternal_
) };
511 static inline SimdFloat gmx_simdcall
cvtI2R(SimdFInt32 a
)
513 return { _mm512_cvtepi32_ps(a
.simdInternal_
) };
516 static inline SimdFIBool gmx_simdcall
cvtB2IB(SimdFBool a
)
518 return { a
.simdInternal_
};
521 static inline SimdFBool gmx_simdcall
cvtIB2B(SimdFIBool a
)
523 return { a
.simdInternal_
};
528 #endif // GMX_SIMD_IMPL_X86_AVX_512_SIMD_FLOAT_H