2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013, by the GROMACS development team, led by
5 * David van der Spoel, Berk Hess, Erik Lindahl, and including many
6 * others, as listed in the AUTHORS file in the top-level source
7 * directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 #ifndef _gmx_x86_avx_128_fma_h_
36 #define _gmx_x86_avx_128_fma_h_
39 #include <immintrin.h>
40 #ifdef HAVE_X86INTRIN_H
41 #include <x86intrin.h> /* FMA */
44 #include <intrin.h> /* FMA MSVC */
50 #include "types/simple.h"
53 #define gmx_mm_extract_epi32(x, imm) _mm_cvtsi128_si32(_mm_srli_si128((x), 4 * (imm)))
55 #define _GMX_MM_BLEND(b3, b2, b1, b0) (((b3) << 3) | ((b2) << 2) | ((b1) << 1) | ((b0)))
57 #define _GMX_MM_PERMUTE128D(fp1, fp0) (((fp1) << 1) | ((fp0)))
60 #define GMX_MM_TRANSPOSE2_PD(row0, row1) { \
61 __m128d __gmx_t1 = row0; \
62 row0 = _mm_unpacklo_pd(row0, row1); \
63 row1 = _mm_unpackhi_pd(__gmx_t1, row1); \
67 #if (defined (_MSC_VER) || defined(__INTEL_COMPILER))
68 # define gmx_mm_castsi128_ps(a) _mm_castsi128_ps(a)
69 # define gmx_mm_castps_si128(a) _mm_castps_si128(a)
70 # define gmx_mm_castps_ps128(a) (a)
71 # define gmx_mm_castsi128_pd(a) _mm_castsi128_pd(a)
72 # define gmx_mm_castpd_si128(a) _mm_castpd_si128(a)
73 #elif defined(__GNUC__)
74 # define gmx_mm_castsi128_ps(a) ((__m128)(a))
75 # define gmx_mm_castps_si128(a) ((__m128i)(a))
76 # define gmx_mm_castps_ps128(a) ((__m128)(a))
77 # define gmx_mm_castsi128_pd(a) ((__m128d)(a))
78 # define gmx_mm_castpd_si128(a) ((__m128i)(a))
80 static __m128
gmx_mm_castsi128_ps(__m128i a
)
82 return *(__m128
*) &a
;
84 static __m128i
gmx_mm_castps_si128(__m128 a
)
86 return *(__m128i
*) &a
;
88 static __m128
gmx_mm_castps_ps128(__m128 a
)
90 return *(__m128
*) &a
;
92 static __m128d
gmx_mm_castsi128_pd(__m128i a
)
94 return *(__m128d
*) &a
;
96 static __m128i
gmx_mm_castpd_si128(__m128d a
)
98 return *(__m128i
*) &a
;
102 #if GMX_EMULATE_AMD_FMA
103 /* Wrapper routines so we can do test builds on non-FMA or non-AMD hardware */
105 _mm_macc_ps(__m128 a
, __m128 b
, __m128 c
)
107 return _mm_add_ps(c
, _mm_mul_ps(a
, b
));
111 _mm_nmacc_ps(__m128 a
, __m128 b
, __m128 c
)
113 return _mm_sub_ps(c
, _mm_mul_ps(a
, b
));
117 _mm_msub_ps(__m128 a
, __m128 b
, __m128 c
)
119 return _mm_sub_ps(_mm_mul_ps(a
, b
), c
);
123 _mm_macc_pd(__m128d a
, __m128d b
, __m128d c
)
125 return _mm_add_pd(c
, _mm_mul_pd(a
, b
));
129 _mm_nmacc_pd(__m128d a
, __m128d b
, __m128d c
)
131 return _mm_sub_pd(c
, _mm_mul_pd(a
, b
));
135 _mm_msub_pd(__m128d a
, __m128d b
, __m128d c
)
137 return _mm_sub_pd(_mm_mul_pd(a
, b
), c
);
139 #endif /* AMD FMA emulation support */
142 gmx_mm_printxmm_ps(const char *s
, __m128 xmm
)
146 _mm_storeu_ps(f
, xmm
);
147 printf("%s: %15.10e %15.10e %15.10e %15.10e\n", s
, f
[0], f
[1], f
[2], f
[3]);
152 gmx_mm_printxmmsum_ps(const char *s
, __m128 xmm
)
156 _mm_storeu_ps(f
, xmm
);
157 printf("%s (sum): %15.10g\n", s
, f
[0]+f
[1]+f
[2]+f
[3]);
162 gmx_mm_printxmm_pd(const char *s
, __m128d xmm
)
166 _mm_storeu_pd(f
, xmm
);
167 printf("%s: %30.20e %30.20e\n", s
, f
[0], f
[1]);
171 gmx_mm_printxmmsum_pd(const char *s
, __m128d xmm
)
175 _mm_storeu_pd(f
, xmm
);
176 printf("%s (sum): %15.10g\n", s
, f
[0]+f
[1]);
181 gmx_mm_printxmm_epi32(const char *s
, __m128i xmmi
)
185 _mm_storeu_si128((__m128i
*)i
, xmmi
);
186 printf("%10s: %2d %2d %2d %2d\n", s
, i
[0], i
[1], i
[2], i
[3]);
191 static int gmx_mm_check_and_reset_overflow(void)
196 MXCSR
= _mm_getcsr();
197 /* The overflow flag is bit 3 in the register */
201 /* Set the overflow flag to zero */
202 MXCSR
= MXCSR
& 0xFFF7;
213 /* Work around gcc bug with wrong type for mask formal parameter to maskload/maskstore */
214 #ifdef GMX_X86_AVX_GCC_MASKLOAD_BUG
215 # define gmx_mm_maskload_ps(mem, mask) _mm_maskload_ps((mem), _mm_castsi128_ps(mask))
216 # define gmx_mm_maskstore_ps(mem, mask, x) _mm_maskstore_ps((mem), _mm_castsi128_ps(mask), (x))
217 # define gmx_mm256_maskload_ps(mem, mask) _mm256_maskload_ps((mem), _mm256_castsi256_ps(mask))
218 # define gmx_mm256_maskstore_ps(mem, mask, x) _mm256_maskstore_ps((mem), _mm256_castsi256_ps(mask), (x))
220 # define gmx_mm_maskload_ps(mem, mask) _mm_maskload_ps((mem), (mask))
221 # define gmx_mm_maskstore_ps(mem, mask, x) _mm_maskstore_ps((mem), (mask), (x))
222 # define gmx_mm256_maskload_ps(mem, mask) _mm256_maskload_ps((mem), (mask))
223 # define gmx_mm256_maskstore_ps(mem, mask, x) _mm256_maskstore_ps((mem), (mask), (x))
228 #endif /* _gmx_x86_avx_128_fma_h_ */