2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013, by the GROMACS development team, led by
5 * David van der Spoel, Berk Hess, Erik Lindahl, and including many
6 * others, as listed in the AUTHORS file in the top-level source
7 * directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 #ifndef _gmx_x86_avx_256_h_
36 #define _gmx_x86_avx_256_h_
39 #include <immintrin.h>
40 #ifdef HAVE_X86INTRIN_H
41 #include <x86intrin.h> /* FMA */
47 #include "types/simple.h"
50 #define gmx_mm_extract_epi32(x, imm) _mm_cvtsi128_si32(_mm_srli_si128((x), 4 * (imm)))
52 #define _GMX_MM_BLEND256D(b3, b2, b1, b0) (((b3) << 3) | ((b2) << 2) | ((b1) << 1) | ((b0)))
53 #define _GMX_MM_PERMUTE(fp3, fp2, fp1, fp0) (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | ((fp0)))
54 #define _GMX_MM_PERMUTE256D(fp3, fp2, fp1, fp0) (((fp3) << 3) | ((fp2) << 2) | ((fp1) << 1) | ((fp0)))
55 #define _GMX_MM_PERMUTE128D(fp1, fp0) (((fp1) << 1) | ((fp0)))
58 #define GMX_MM_TRANSPOSE2_PD(row0, row1) { \
59 __m128d __gmx_t1 = row0; \
60 row0 = _mm_unpacklo_pd(row0, row1); \
61 row1 = _mm_unpackhi_pd(__gmx_t1, row1); \
64 #define GMX_MM256_FULLTRANSPOSE4_PD(row0, row1, row2, row3) \
66 __m256d _t0, _t1, _t2, _t3; \
67 _t0 = _mm256_unpacklo_pd((row0), (row1)); \
68 _t1 = _mm256_unpackhi_pd((row0), (row1)); \
69 _t2 = _mm256_unpacklo_pd((row2), (row3)); \
70 _t3 = _mm256_unpackhi_pd((row2), (row3)); \
71 row0 = _mm256_permute2f128_pd(_t0, _t2, 0x20); \
72 row1 = _mm256_permute2f128_pd(_t1, _t3, 0x20); \
73 row2 = _mm256_permute2f128_pd(_t0, _t2, 0x31); \
74 row3 = _mm256_permute2f128_pd(_t1, _t3, 0x31); \
77 #if (defined (_MSC_VER) || defined(__INTEL_COMPILER))
78 # define gmx_mm_castsi128_ps(a) _mm_castsi128_ps(a)
79 # define gmx_mm_castps_si128(a) _mm_castps_si128(a)
80 # define gmx_mm_castps_ps128(a) (a)
81 # define gmx_mm_castsi128_pd(a) _mm_castsi128_pd(a)
82 # define gmx_mm_castpd_si128(a) _mm_castpd_si128(a)
83 #elif defined(__GNUC__)
84 # define gmx_mm_castsi128_ps(a) ((__m128)(a))
85 # define gmx_mm_castps_si128(a) ((__m128i)(a))
86 # define gmx_mm_castps_ps128(a) ((__m128)(a))
87 # define gmx_mm_castsi128_pd(a) ((__m128d)(a))
88 # define gmx_mm_castpd_si128(a) ((__m128i)(a))
90 static __m128
gmx_mm_castsi128_ps(__m128i a
)
92 return *(__m128
*) &a
;
94 static __m128i
gmx_mm_castps_si128(__m128 a
)
96 return *(__m128i
*) &a
;
98 static __m128
gmx_mm_castps_ps128(__m128 a
)
100 return *(__m128
*) &a
;
102 static __m128d
gmx_mm_castsi128_pd(__m128i a
)
104 return *(__m128d
*) &a
;
106 static __m128i
gmx_mm_castpd_si128(__m128d a
)
108 return *(__m128i
*) &a
;
112 static gmx_inline __m256
113 gmx_mm256_unpack128lo_ps(__m256 xmm1
, __m256 xmm2
)
115 return _mm256_permute2f128_ps(xmm1
, xmm2
, 0x20);
118 static gmx_inline __m256
119 gmx_mm256_unpack128hi_ps(__m256 xmm1
, __m256 xmm2
)
121 return _mm256_permute2f128_ps(xmm1
, xmm2
, 0x31);
124 static gmx_inline __m256
125 gmx_mm256_set_m128(__m128 hi
, __m128 lo
)
127 return _mm256_insertf128_ps(_mm256_castps128_ps256(lo
), hi
, 0x1);
131 static gmx_inline __m256
132 gmx_mm256_load4_ps(float const * p
)
137 return _mm256_insertf128_ps(_mm256_castps128_ps256(a
), a
, 0x1);
142 gmx_mm256_unpack128lo_pd(__m256d xmm1
, __m256d xmm2
)
144 return _mm256_permute2f128_pd(xmm1
, xmm2
, 0x20);
148 gmx_mm256_unpack128hi_pd(__m256d xmm1
, __m256d xmm2
)
150 return _mm256_permute2f128_pd(xmm1
, xmm2
, 0x31);
154 gmx_mm256_set_m128d(__m128d hi
, __m128d lo
)
156 return _mm256_insertf128_pd(_mm256_castpd128_pd256(lo
), hi
, 0x1);
160 static __m128
gmx_mm256_sum4h_m128(__m256 x
, __m256 y
)
164 sum
= _mm256_add_ps(x
, y
);
165 return _mm_add_ps(_mm256_castps256_ps128(sum
), _mm256_extractf128_ps(sum
, 0x1));
170 gmx_mm_printxmm_ps(const char *s
, __m128 xmm
)
174 _mm_storeu_ps(f
, xmm
);
175 printf("%s: %15.10e %15.10e %15.10e %15.10e\n", s
, f
[0], f
[1], f
[2], f
[3]);
180 gmx_mm_printxmmsum_ps(const char *s
, __m128 xmm
)
184 _mm_storeu_ps(f
, xmm
);
185 printf("%s (sum): %15.10g\n", s
, f
[0]+f
[1]+f
[2]+f
[3]);
190 gmx_mm_printxmm_pd(const char *s
, __m128d xmm
)
194 _mm_storeu_pd(f
, xmm
);
195 printf("%s: %30.20e %30.20e\n", s
, f
[0], f
[1]);
199 gmx_mm_printxmmsum_pd(const char *s
, __m128d xmm
)
203 _mm_storeu_pd(f
, xmm
);
204 printf("%s (sum): %15.10g\n", s
, f
[0]+f
[1]);
209 gmx_mm_printxmm_epi32(const char *s
, __m128i xmmi
)
213 _mm_storeu_si128((__m128i
*)i
, xmmi
);
214 printf("%10s: %2d %2d %2d %2d\n", s
, i
[0], i
[1], i
[2], i
[3]);
218 gmx_mm256_printymm_ps(const char *s
, __m256 ymm
)
222 _mm256_storeu_ps(f
, ymm
);
223 printf("%s: %12.7f %12.7f %12.7f %12.7f %12.7f %12.7f %12.7f %12.7f\n", s
, f
[0], f
[1], f
[2], f
[3], f
[4], f
[5], f
[6], f
[7]);
227 gmx_mm256_printymmsum_ps(const char *s
, __m256 ymm
)
231 _mm256_storeu_ps(f
, ymm
);
232 printf("%s (sum): %15.10g\n", s
, f
[0]+f
[1]+f
[2]+f
[3]+f
[4]+f
[5]+f
[6]+f
[7]);
237 gmx_mm256_printymm_pd(const char *s
, __m256d ymm
)
241 _mm256_storeu_pd(f
, ymm
);
242 printf("%s: %16.12f %16.12f %16.12f %16.12f\n", s
, f
[0], f
[1], f
[2], f
[3]);
246 gmx_mm256_printymmsum_pd(const char *s
, __m256d ymm
)
250 _mm256_storeu_pd(f
, ymm
);
251 printf("%s (sum): %15.10g\n", s
, f
[0]+f
[1]+f
[2]+f
[3]);
257 gmx_mm256_printymm_epi32(const char *s
, __m256i ymmi
)
261 _mm256_storeu_si256((__m256i
*)i
, ymmi
);
262 printf("%10s: %2d %2d %2d %2d %2d %2d %2d %2d\n", s
, i
[0], i
[1], i
[2], i
[3], i
[4], i
[5], i
[6], i
[7]);
267 static int gmx_mm_check_and_reset_overflow(void)
272 MXCSR
= _mm_getcsr();
273 /* The overflow flag is bit 3 in the register */
277 /* Set the overflow flag to zero */
278 MXCSR
= MXCSR
& 0xFFF7;
289 /* Work around gcc bug with wrong type for mask formal parameter to maskload/maskstore */
290 #ifdef GMX_X86_AVX_GCC_MASKLOAD_BUG
291 # define gmx_mm_maskload_ps(mem, mask) _mm_maskload_ps((mem), _mm_castsi128_ps(mask))
292 # define gmx_mm_maskstore_ps(mem, mask, x) _mm_maskstore_ps((mem), _mm_castsi128_ps(mask), (x))
293 # define gmx_mm256_maskload_ps(mem, mask) _mm256_maskload_ps((mem), _mm256_castsi256_ps(mask))
294 # define gmx_mm256_maskstore_ps(mem, mask, x) _mm256_maskstore_ps((mem), _mm256_castsi256_ps(mask), (x))
296 # define gmx_mm_maskload_ps(mem, mask) _mm_maskload_ps((mem), (mask))
297 # define gmx_mm_maskstore_ps(mem, mask, x) _mm_maskstore_ps((mem), (mask), (x))
298 # define gmx_mm256_maskload_ps(mem, mask) _mm256_maskload_ps((mem), (mask))
299 # define gmx_mm256_maskstore_ps(mem, mask, x) _mm256_maskstore_ps((mem), (mask), (x))
303 #endif /* _gmx_x86_avx_256_h_ */