include/gmx_x86_avx_256.h

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012, by the GROMACS development team, led by
   5  * David van der Spoel, Berk Hess, Erik Lindahl, and including many
   6  * others, as listed in the AUTHORS file in the top-level source
   7  * directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 #ifndef _gmx_x86_avx_256_h_
  36 #define _gmx_x86_avx_256_h_
  37
  38
  39 #include <immintrin.h>
  40 #ifdef HAVE_X86INTRIN_H
  41 #include <x86intrin.h> /* FMA */
  42 #endif
  43
  44
  45 #include <stdio.h>
  46
  47 #include "types/simple.h"
  48
  49
  50 #define gmx_mm_extract_epi32(x, imm) _mm_cvtsi128_si32(_mm_srli_si128((x), 4 * (imm)))
  51
  52 #define _GMX_MM_BLEND256D(b3,b2,b1,b0) (((b3) << 3) | ((b2) << 2) | ((b1) << 1) | ((b0)))
  53 #define _GMX_MM_PERMUTE(fp3,fp2,fp1,fp0) (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | ((fp0)))
  54 #define _GMX_MM_PERMUTE256D(fp3,fp2,fp1,fp0) (((fp3) << 3) | ((fp2) << 2) | ((fp1) << 1) | ((fp0)))
  55 #define _GMX_MM_PERMUTE128D(fp1,fp0)         (((fp1) << 1) | ((fp0)))
  56
  57
  58 #define GMX_MM_TRANSPOSE2_PD(row0, row1) {           \
  59     __m128d __gmx_t1 = row0;                         \
  60     row0           = _mm_unpacklo_pd(row0,row1);     \
  61     row1           = _mm_unpackhi_pd(__gmx_t1,row1); \
  62 }
  63
  64 #define GMX_MM256_FULLTRANSPOSE4_PD(row0,row1,row2,row3) \
  65 {                                                        \
  66     __m256d _t0, _t1, _t2, _t3;                          \
  67     _t0  = _mm256_unpacklo_pd((row0), (row1));           \
  68     _t1  = _mm256_unpackhi_pd((row0), (row1));           \
  69     _t2  = _mm256_unpacklo_pd((row2), (row3));           \
  70     _t3  = _mm256_unpackhi_pd((row2), (row3));           \
  71     row0 = _mm256_permute2f128_pd(_t0, _t2, 0x20);       \
  72     row1 = _mm256_permute2f128_pd(_t1, _t3, 0x20);       \
  73     row2 = _mm256_permute2f128_pd(_t0, _t2, 0x31);       \
  74     row3 = _mm256_permute2f128_pd(_t1, _t3, 0x31);       \
  75 }
  76
  77 #if (defined (_MSC_VER) || defined(__INTEL_COMPILER))
  78 #  define gmx_mm_castsi128_ps(a) _mm_castsi128_ps(a)
  79 #  define gmx_mm_castps_si128(a) _mm_castps_si128(a)
  80 #  define gmx_mm_castps_ps128(a) (a)
  81 #  define gmx_mm_castsi128_pd(a) _mm_castsi128_pd(a)
  82 #  define gmx_mm_castpd_si128(a) _mm_castpd_si128(a)
  83 #elif defined(__GNUC__)
  84 #  define gmx_mm_castsi128_ps(a) ((__m128)(a))
  85 #  define gmx_mm_castps_si128(a) ((__m128i)(a))
  86 #  define gmx_mm_castps_ps128(a) ((__m128)(a))
  87 #  define gmx_mm_castsi128_pd(a) ((__m128d)(a))
  88 #  define gmx_mm_castpd_si128(a) ((__m128i)(a))
  89 #else
  90 static __m128  gmx_mm_castsi128_ps(__m128i a)
  91 {
  92     return *(__m128 *) &a;
  93 }
  94 static __m128i gmx_mm_castps_si128(__m128 a)
  95 {
  96     return *(__m128i *) &a;
  97 }
  98 static __m128  gmx_mm_castps_ps128(__m128 a)
  99 {
 100     return *(__m128 *) &a;
 101 }
 102 static __m128d gmx_mm_castsi128_pd(__m128i a)
 103 {
 104     return *(__m128d *) &a;
 105 }
 106 static __m128i gmx_mm_castpd_si128(__m128d a)
 107 {
 108     return *(__m128i *) &a;
 109 }
 110 #endif
 111
 112 static gmx_inline __m256
 113 gmx_mm256_unpack128lo_ps(__m256 xmm1, __m256 xmm2)
 114 {
 115     return _mm256_permute2f128_ps(xmm1,xmm2,0x20);
 116 }
 117
 118 static gmx_inline __m256
 119 gmx_mm256_unpack128hi_ps(__m256 xmm1, __m256 xmm2)
 120 {
 121     return _mm256_permute2f128_ps(xmm1,xmm2,0x31);
 122 }
 123
 124 static gmx_inline __m256
 125 gmx_mm256_set_m128(__m128 hi, __m128 lo)
 126 {
 127     return _mm256_insertf128_ps(_mm256_castps128_ps256(lo), hi, 0x1);
 128 }
 129
 130
 131 static gmx_inline __m256
 132 gmx_mm256_load4_ps(float const * p)
 133 {
 134     __m128 a;
 135
 136     a = _mm_load_ps(p);
 137     return _mm256_insertf128_ps(_mm256_castps128_ps256(a), a, 0x1);
 138 }
 139
 140
 141 static __m256d
 142 gmx_mm256_unpack128lo_pd(__m256d xmm1, __m256d xmm2)
 143 {
 144     return _mm256_permute2f128_pd(xmm1,xmm2,0x20);
 145 }
 146
 147 static __m256d
 148 gmx_mm256_unpack128hi_pd(__m256d xmm1, __m256d xmm2)
 149 {
 150     return _mm256_permute2f128_pd(xmm1,xmm2,0x31);
 151 }
 152
 153 static __m256d
 154 gmx_mm256_set_m128d(__m128d hi, __m128d lo)
 155 {
 156     return _mm256_insertf128_pd(_mm256_castpd128_pd256(lo), hi, 0x1);
 157 }
 158
 159
 160 static __m128 gmx_mm256_sum4h_m128(__m256 x, __m256 y)
 161 {
 162     __m256 sum;
 163
 164     sum = _mm256_add_ps(x,y);
 165     return _mm_add_ps(_mm256_castps256_ps128(sum),_mm256_extractf128_ps(sum,0x1));
 166 }
 167
 168
 169 static void
 170 gmx_mm_printxmm_ps(const char *s,__m128 xmm)
 171 {
 172     float f[4];
 173
 174     _mm_storeu_ps(f,xmm);
 175     printf("%s: %15.10e %15.10e %15.10e %15.10e\n",s,f[0],f[1],f[2],f[3]);
 176 }
 177
 178
 179 static void
 180 gmx_mm_printxmmsum_ps(const char *s,__m128 xmm)
 181 {
 182     float f[4];
 183
 184     _mm_storeu_ps(f,xmm);
 185     printf("%s (sum): %15.10g\n",s,f[0]+f[1]+f[2]+f[3]);
 186 }
 187
 188
 189 static void
 190 gmx_mm_printxmm_pd(const char *s,__m128d xmm)
 191 {
 192     double f[2];
 193
 194     _mm_storeu_pd(f,xmm);
 195     printf("%s: %30.20e %30.20e\n",s,f[0],f[1]);
 196 }
 197
 198 static void
 199 gmx_mm_printxmmsum_pd(const char *s,__m128d xmm)
 200 {
 201     double f[2];
 202
 203     _mm_storeu_pd(f,xmm);
 204     printf("%s (sum): %15.10g\n",s,f[0]+f[1]);
 205 }
 206
 207
 208 static void
 209 gmx_mm_printxmm_epi32(const char *s,__m128i xmmi)
 210 {
 211     int i[4];
 212
 213     _mm_storeu_si128((__m128i *)i,xmmi);
 214     printf("%10s: %2d %2d %2d %2d\n",s,i[0],i[1],i[2],i[3]);
 215 }
 216
 217 static void
 218 gmx_mm256_printymm_ps(const char *s,__m256 ymm)
 219 {
 220     float f[8];
 221
 222     _mm256_storeu_ps(f,ymm);
 223     printf("%s: %12.7f %12.7f %12.7f %12.7f %12.7f %12.7f %12.7f %12.7f\n",s,f[0],f[1],f[2],f[3],f[4],f[5],f[6],f[7]);
 224 }
 225
 226 static void
 227 gmx_mm256_printymmsum_ps(const char *s,__m256 ymm)
 228 {
 229     float f[8];
 230
 231     _mm256_storeu_ps(f,ymm);
 232     printf("%s (sum): %15.10g\n",s,f[0]+f[1]+f[2]+f[3]+f[4]+f[5]+f[6]+f[7]);
 233 }
 234
 235
 236 static void
 237 gmx_mm256_printymm_pd(const char *s,__m256d ymm)
 238 {
 239     double f[4];
 240
 241     _mm256_storeu_pd(f,ymm);
 242     printf("%s: %16.12f %16.12f %16.12f %16.12f\n",s,f[0],f[1],f[2],f[3]);
 243 }
 244
 245 static void
 246 gmx_mm256_printymmsum_pd(const char *s,__m256d ymm)
 247 {
 248     double f[4];
 249
 250     _mm256_storeu_pd(f,ymm);
 251     printf("%s (sum): %15.10g\n",s,f[0]+f[1]+f[2]+f[3]);
 252 }
 253
 254
 255
 256 static void
 257 gmx_mm256_printymm_epi32(const char *s,__m256i ymmi)
 258 {
 259     int i[8];
 260
 261     _mm256_storeu_si256((__m256i *)i,ymmi);
 262     printf("%10s: %2d %2d %2d %2d %2d %2d %2d %2d\n",s,i[0],i[1],i[2],i[3],i[4],i[5],i[6],i[7]);
 263 }
 264
 265
 266
 267 static int gmx_mm_check_and_reset_overflow(void)
 268 {
 269     int MXCSR;
 270     int sse_overflow;
 271
 272     MXCSR = _mm_getcsr();
 273     /* The overflow flag is bit 3 in the register */
 274     if (MXCSR & 0x0008)
 275     {
 276         sse_overflow = 1;
 277         /* Set the overflow flag to zero */
 278         MXCSR = MXCSR & 0xFFF7;
 279         _mm_setcsr(MXCSR);
 280     }
 281     else
 282     {
 283         sse_overflow = 0;
 284     }
 285
 286     return sse_overflow;
 287 }
 288
 289 /* Work around gcc bug with wrong type for mask formal parameter to maskload/maskstore */
 290 #ifdef GMX_X86_AVX_GCC_MASKLOAD_BUG
 291 #    define gmx_mm_maskload_ps(mem,mask)       _mm_maskload_ps((mem),_mm_castsi128_ps(mask))
 292 #    define gmx_mm_maskstore_ps(mem,mask,x)    _mm_maskstore_ps((mem),_mm_castsi128_ps(mask),(x))
 293 #    define gmx_mm256_maskload_ps(mem,mask)    _mm256_maskload_ps((mem),_mm256_castsi256_ps(mask))
 294 #    define gmx_mm256_maskstore_ps(mem,mask,x) _mm256_maskstore_ps((mem),_mm256_castsi256_ps(mask),(x))
 295 #else
 296 #    define gmx_mm_maskload_ps(mem,mask)       _mm_maskload_ps((mem),(mask))
 297 #    define gmx_mm_maskstore_ps(mem,mask,x)    _mm_maskstore_ps((mem),(mask),(x))
 298 #    define gmx_mm256_maskload_ps(mem,mask)    _mm256_maskload_ps((mem),(mask))
 299 #    define gmx_mm256_maskstore_ps(mem,mask,x) _mm256_maskstore_ps((mem),(mask),(x))
 300 #endif
 301
 302
 303 #endif /* _gmx_x86_avx_256_h_ */