?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
added Verlet scheme and NxN non-bonded functionality
[gromacs.git] / src / mdlib / nbnxn_kernels / nbnxn_kernel_x86_simd_utils.h
blob4ef461092254bdb46abbae7229bb75f4f11f7ba5
1 /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
2 *
3 *
4 * This source code is part of
5 *
6 * G R O M A C S
7 *
8 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
9 * Copyright (c) 2001-2012, The GROMACS Development Team
10 *
11 * Gromacs is a library for molecular simulation and trajectory analysis,
12 * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
13 * a full list of developers and information, check out http://www.gromacs.org
14 *
15 * This program is free software; you can redistribute it and/or modify it under
16 * the terms of the GNU Lesser General Public License as published by the Free
17 * Software Foundation; either version 2 of the License, or (at your option) any
18 * later version.
19 * As a special exception, you may use this file as part of a free software
20 * library without restriction. Specifically, if other files instantiate
21 * templates or use macros or inline functions from this file, or you compile
22 * this file and link it with other files to produce an executable, this
23 * file does not by itself cause the resulting executable to be covered by
24 * the GNU Lesser General Public License.
25 *
26 * In plain-speak: do not worry about classes/macros/templates either - only
27 * changes to the library have to be LGPL, not an application linking with it.
28 *
29 * To help fund GROMACS development, we humbly ask that you cite
30 * the papers people have written on it - you can find them on the website!
31 */
32 #ifndef _nbnxn_kernel_sse_utils_h_
33 #define _nbnxn_kernel_sse_utils_h_
34
35 /* This files contains all functions/macros for the SSE/AVX kernels
36 * which have explicit dependencies on the j-size / SIMD-width, which
37 * can be 2 (SSE-double), 4 (SSE-single,AVX-double) or 8 (AVX-single).
38 * The functionality which depends on the j-cluster size is:
39 * LJ-parameter lookup
40 * force table lookup
41 * energy group pair energy storage
42 */
43
44 #define GMX_MM_TRANSPOSE2_OP_PD(in0,in1,out0,out1) \
45 { \
46 out0 = _mm_shuffle_pd(in0,in1,_MM_SHUFFLE2(0,0)); \
47 out1 = _mm_shuffle_pd(in0,in1,_MM_SHUFFLE2(1,1)); \
48 }
49
50 #if defined GMX_MM128_HERE || !defined GMX_DOUBLE
51 #define GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(in0,in1,in2,in3,out0,out1) \
52 { \
53 __m128 _c01,_c23; \
54 _c01 = _mm_shuffle_ps(in0,in1,_MM_SHUFFLE(1,0,1,0)); \
55 _c23 = _mm_shuffle_ps(in2,in3,_MM_SHUFFLE(1,0,1,0)); \
56 out0 = _mm_shuffle_ps(_c01,_c23,_MM_SHUFFLE(2,0,2,0)); \
57 out1 = _mm_shuffle_ps(_c01,_c23,_MM_SHUFFLE(3,1,3,1)); \
58 }
59 #else
60 #define GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(in0,in1,in2,in3,out0,out1) \
61 { \
62 __m256d _c01,_c23; \
63 _c01 = _mm256_shuffle_pd(in0,in1,_MM_SHUFFLE(1,0,1,0)); \
64 _c23 = _mm256_shuffle_pd(in2,in3,_MM_SHUFFLE(1,0,1,0)); \
65 out0 = _mm256_shuffle_pd(_c01,_c23,_MM_SHUFFLE(2,0,2,0)); \
66 out1 = _mm256_shuffle_pd(_c01,_c23,_MM_SHUFFLE(3,1,3,1)); \
67 }
68 #endif
69
70 #define GMX_MM_SHUFFLE_4_PS_FIL2_TO_1_PS(in0,in1,in2,in3,out) \
71 { \
72 __m128 _c01,_c23; \
73 _c01 = _mm_shuffle_ps(in0,in1,_MM_SHUFFLE(3,2,3,2)); \
74 _c23 = _mm_shuffle_ps(in2,in3,_MM_SHUFFLE(3,2,3,2)); \
75 out = _mm_shuffle_ps(_c01,_c23,_MM_SHUFFLE(2,0,2,0)); \
76 }
77
78 #ifndef GMX_MM256_HERE
79 #ifndef GMX_DOUBLE
80 #define GMX_MM_TRANSPOSE_SUM4_PR(i_SSE0,i_SSE1,i_SSE2,i_SSE3,o_SSE) \
81 { \
82 _MM_TRANSPOSE4_PS(i_SSE0,i_SSE1,i_SSE2,i_SSE3); \
83 i_SSE0 = _mm_add_ps(i_SSE0,i_SSE1); \
84 i_SSE2 = _mm_add_ps(i_SSE2,i_SSE3); \
85 o_SSE = _mm_add_ps(i_SSE0,i_SSE2); \
86 }
87 #else
88 #define GMX_MM_TRANSPOSE_SUM2_PD(i_SSE0,i_SSE1,o_SSE) \
89 { \
90 GMX_MM_TRANSPOSE2_PD(i_SSE0,i_SSE1); \
91 o_SSE = _mm_add_pd(i_SSE0,i_SSE1); \
92 }
93 #endif
94 #else
95 #ifndef GMX_DOUBLE
96 #define GMX_MM_TRANSPOSE_SUM4_PR(i_SSE0,i_SSE1,i_SSE2,i_SSE3,o_SSE) \
97 { \
98 i_SSE0 = _mm256_hadd_ps(i_SSE0,i_SSE1); \
99 i_SSE2 = _mm256_hadd_ps(i_SSE2,i_SSE3); \
100 i_SSE1 = _mm256_hadd_ps(i_SSE0,i_SSE2); \
101 o_SSE = _mm_add_ps(_mm256_castps256_ps128(i_SSE1),_mm256_extractf128_ps(i_SSE1,1)); \
102 }
103 #else
104 #define GMX_MM_TRANSPOSE_SUM4_PR(i_SSE0,i_SSE1,i_SSE2,i_SSE3,o_SSE) \
105 { \
106 i_SSE0 = _mm256_hadd_pd(i_SSE0,i_SSE1); \
107 i_SSE2 = _mm256_hadd_pd(i_SSE2,i_SSE3); \
108 o_SSE = _mm256_add_pd(_mm256_permute2f128_pd(i_SSE0,i_SSE2,0x20),_mm256_permute2f128_pd(i_SSE0,i_SSE2,0x31)); \
109 }
110 #endif
111 #endif
112
113 #ifdef GMX_MM128_HERE
114
115 static inline __m128
116 gmx_mm128_invsqrt_ps_single(__m128 x)
117 {
118 const __m128 half = _mm_set_ps(0.5,0.5,0.5,0.5);
119 const __m128 three = _mm_set_ps(3.0,3.0,3.0,3.0);
120
121 __m128 lu = _mm_rsqrt_ps(x);
122
123 return _mm_mul_ps(half,_mm_mul_ps(_mm_sub_ps(three,_mm_mul_ps(_mm_mul_ps(lu,lu),x)),lu));
124 }
125
126 /* Do 2/4 double precision invsqrt operations.
127 * Doing the SSE rsqrt and the first Newton Raphson iteration
128 * in single precision gives full double precision accuracy.
129 * The speed is more than twice as fast as two gmx_mm_invsqrt_pd calls.
130 */
131 #define GMX_MM128_INVSQRT2_PD(i_SSE0,i_SSE1,o_SSE0,o_SSE1) \
132 { \
133 const __m128d half = _mm_set1_pd(0.5); \
134 const __m128d three = _mm_set1_pd(3.0); \
135 __m128 s_SSE,ir_SSE; \
136 __m128d lu0,lu1; \
137 \
138 s_SSE = _mm_movelh_ps(_mm_cvtpd_ps(i_SSE0),_mm_cvtpd_ps(i_SSE1)); \
139 ir_SSE = gmx_mm128_invsqrt_ps_single(s_SSE); \
140 lu0 = _mm_cvtps_pd(ir_SSE); \
141 lu1 = _mm_cvtps_pd(_mm_movehl_ps(ir_SSE,ir_SSE)); \
142 o_SSE0 = _mm_mul_pd(half,_mm_mul_pd(_mm_sub_pd(three,_mm_mul_pd(_mm_mul_pd(lu0,lu0),i_SSE0)),lu0)); \
143 o_SSE1 = _mm_mul_pd(half,_mm_mul_pd(_mm_sub_pd(three,_mm_mul_pd(_mm_mul_pd(lu1,lu1),i_SSE1)),lu1)); \
144 }
145
146 #define GMX_MM_INVSQRT2_PD GMX_MM128_INVSQRT2_PD
147
148 #endif
149
150 #ifdef GMX_MM256_HERE
151
152 static inline __m256
153 gmx_mm256_invsqrt_ps_single(__m256 x)
154 {
155 const __m256 half = _mm256_set_ps(0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5);
156 const __m256 three = _mm256_set_ps(3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0);
157
158 __m256 lu = _mm256_rsqrt_ps(x);
159
160 return _mm256_mul_ps(half,_mm256_mul_ps(_mm256_sub_ps(three,_mm256_mul_ps(_mm256_mul_ps(lu,lu),x)),lu));
161 }
162
163 #define GMX_MM256_INVSQRT2_PD(i_SSE0,i_SSE1,o_SSE0,o_SSE1) \
164 { \
165 const __m256d half = _mm256_set1_pd(0.5); \
166 const __m256d three = _mm256_set1_pd(3.0); \
167 __m256 s_SSE,ir_SSE; \
168 __m256d lu0,lu1; \
169 \
170 s_SSE = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm256_cvtpd_ps(i_SSE0)),_mm256_cvtpd_ps(i_SSE1),1); \
171 ir_SSE = gmx_mm256_invsqrt_ps_single(s_SSE); \
172 lu0 = _mm256_cvtps_pd(_mm256_castps256_ps128(ir_SSE)); \
173 lu1 = _mm256_cvtps_pd(_mm256_extractf128_ps(ir_SSE,1)); \
174 o_SSE0 = _mm256_mul_pd(half,_mm256_mul_pd(_mm256_sub_pd(three,_mm256_mul_pd(_mm256_mul_pd(lu0,lu0),i_SSE0)),lu0)); \
175 o_SSE1 = _mm256_mul_pd(half,_mm256_mul_pd(_mm256_sub_pd(three,_mm256_mul_pd(_mm256_mul_pd(lu1,lu1),i_SSE1)),lu1)); \
176 }
177
178 #define GMX_MM_INVSQRT2_PD GMX_MM256_INVSQRT2_PD
179
180 #endif
181
182 /* Force and energy table load and interpolation routines */
183
184 #if defined GMX_MM128_HERE && !defined GMX_DOUBLE
185
186 #define load_lj_pair_params(nbfp,type,aj,c6_SSE,c12_SSE) \
187 { \
188 gmx_mm_pr clj_SSE[UNROLLJ]; \
189 int p; \
190 \
191 for(p=0; p<UNROLLJ; p++) \
192 { \
193 /* Here we load 4 aligned floats, but we need just 2 */ \
194 clj_SSE[p] = gmx_load_pr(nbfp+type[aj+p]*NBFP_STRIDE); \
195 } \
196 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(clj_SSE[0],clj_SSE[1],clj_SSE[2],clj_SSE[3],c6_SSE,c12_SSE); \
197 }
198
199 #endif
200
201 #if defined GMX_MM256_HERE && !defined GMX_DOUBLE
202
203 /* Put two 128-bit 4-float registers into one 256-bit 8-float register */
204 #define GMX_2_MM_TO_M256(in0,in1,out) \
205 { \
206 out = _mm256_insertf128_ps(_mm256_castps128_ps256(in0),in1,1); \
207 }
208
209 #define load_lj_pair_params(nbfp,type,aj,c6_SSE,c12_SSE) \
210 { \
211 __m128 clj_SSE[UNROLLJ],c6t_SSE[2],c12t_SSE[2]; \
212 int p; \
213 \
214 for(p=0; p<UNROLLJ; p++) \
215 { \
216 /* Here we load 4 aligned floats, but we need just 2 */ \
217 clj_SSE[p] = _mm_load_ps(nbfp+type[aj+p]*NBFP_STRIDE); \
218 } \
219 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(clj_SSE[0],clj_SSE[1],clj_SSE[2],clj_SSE[3],c6t_SSE[0],c12t_SSE[0]); \
220 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(clj_SSE[4],clj_SSE[5],clj_SSE[6],clj_SSE[7],c6t_SSE[1],c12t_SSE[1]); \
221 \
222 GMX_2_MM_TO_M256(c6t_SSE[0],c6t_SSE[1],c6_SSE); \
223 GMX_2_MM_TO_M256(c12t_SSE[0],c12t_SSE[1],c12_SSE); \
224 }
225
226 #endif
227
228 #if defined GMX_MM128_HERE && defined GMX_DOUBLE
229
230 #define load_lj_pair_params(nbfp,type,aj,c6_SSE,c12_SSE) \
231 { \
232 gmx_mm_pr clj_SSE[UNROLLJ]; \
233 int p; \
234 \
235 for(p=0; p<UNROLLJ; p++) \
236 { \
237 clj_SSE[p] = gmx_load_pr(nbfp+type[aj+p]*NBFP_STRIDE); \
238 } \
239 GMX_MM_TRANSPOSE2_OP_PD(clj_SSE[0],clj_SSE[1],c6_SSE,c12_SSE); \
240 }
241
242 #endif
243
244 #if defined GMX_MM256_HERE && defined GMX_DOUBLE
245
246 #define load_lj_pair_params(nbfp,type,aj,c6_SSE,c12_SSE) \
247 { \
248 __m128d clj_SSE[UNROLLJ],c6t_SSE[2],c12t_SSE[2]; \
249 int p; \
250 \
251 for(p=0; p<UNROLLJ; p++) \
252 { \
253 clj_SSE[p] = _mm_load_pd(nbfp+type[aj+p]*NBFP_STRIDE); \
254 } \
255 GMX_MM_TRANSPOSE2_OP_PD(clj_SSE[0],clj_SSE[1],c6t_SSE[0],c12t_SSE[0]); \
256 GMX_MM_TRANSPOSE2_OP_PD(clj_SSE[2],clj_SSE[3],c6t_SSE[1],c12t_SSE[1]); \
257 GMX_2_M128D_TO_M256D(c6t_SSE[0],c6t_SSE[1],c6_SSE); \
258 GMX_2_M128D_TO_M256D(c12t_SSE[0],c12t_SSE[1],c12_SSE); \
259 }
260
261 #endif
262
263
264 /* The load_table functions below are performance critical.
265 * The routines issue UNROLLI*UNROLLJ _mm_load_ps calls.
266 * As these all have latencies, scheduling is crucial.
267 * The Intel compilers and CPUs seem to do a good job at this.
268 * But AMD CPUs perform significantly worse with gcc than with icc.
269 * Performance is improved a bit by using the extract function UNROLLJ times,
270 * instead of doing an _mm_store_si128 for every i-particle.
271 * With AVX this significantly deteriorates performance (8 extracts iso 4).
272 * Because of this, the load_table_f macro always takes the ti parameter,
273 * but it is only used with AVX.
274 */
275
276 #if defined GMX_MM128_HERE && !defined GMX_DOUBLE
277
278 #define load_table_f(tab_coul_FDV0, ti_SSE, ti, ctab0_SSE, ctab1_SSE) \
279 { \
280 int idx[4]; \
281 __m128 ctab_SSE[4]; \
282 \
283 /* Table has 4 entries, left-shift index by 2 */ \
284 ti_SSE = _mm_slli_epi32(ti_SSE,2); \
285 /* Without SSE4.1 the extract macro needs an immediate: unroll */ \
286 idx[0] = gmx_mm_extract_epi32(ti_SSE,0); \
287 ctab_SSE[0] = _mm_load_ps(tab_coul_FDV0+idx[0]); \
288 idx[1] = gmx_mm_extract_epi32(ti_SSE,1); \
289 ctab_SSE[1] = _mm_load_ps(tab_coul_FDV0+idx[1]); \
290 idx[2] = gmx_mm_extract_epi32(ti_SSE,2); \
291 ctab_SSE[2] = _mm_load_ps(tab_coul_FDV0+idx[2]); \
292 idx[3] = gmx_mm_extract_epi32(ti_SSE,3); \
293 ctab_SSE[3] = _mm_load_ps(tab_coul_FDV0+idx[3]); \
294 \
295 /* Shuffle the force table entries to a convenient order */ \
296 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(ctab_SSE[0],ctab_SSE[1],ctab_SSE[2],ctab_SSE[3],ctab0_SSE,ctab1_SSE); \
297 }
298
299 #define load_table_f_v(tab_coul_FDV0, ti_SSE, ti, ctab0_SSE, ctab1_SSE, ctabv_SSE) \
300 { \
301 int idx[4]; \
302 __m128 ctab_SSE[4]; \
303 \
304 /* Table has 4 entries, left-shift index by 2 */ \
305 ti_SSE = _mm_slli_epi32(ti_SSE,2); \
306 /* Without SSE4.1 the extract macro needs an immediate: unroll */ \
307 idx[0] = gmx_mm_extract_epi32(ti_SSE,0); \
308 ctab_SSE[0] = _mm_load_ps(tab_coul_FDV0+idx[0]); \
309 idx[1] = gmx_mm_extract_epi32(ti_SSE,1); \
310 ctab_SSE[1] = _mm_load_ps(tab_coul_FDV0+idx[1]); \
311 idx[2] = gmx_mm_extract_epi32(ti_SSE,2); \
312 ctab_SSE[2] = _mm_load_ps(tab_coul_FDV0+idx[2]); \
313 idx[3] = gmx_mm_extract_epi32(ti_SSE,3); \
314 ctab_SSE[3] = _mm_load_ps(tab_coul_FDV0+idx[3]); \
315 \
316 /* Shuffle the force table entries to a convenient order */ \
317 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(ctab_SSE[0],ctab_SSE[1],ctab_SSE[2],ctab_SSE[3],ctab0_SSE,ctab1_SSE); \
318 /* Shuffle the energy table entries to a convenient order */ \
319 GMX_MM_SHUFFLE_4_PS_FIL2_TO_1_PS(ctab_SSE[0],ctab_SSE[1],ctab_SSE[2],ctab_SSE[3],ctabv_SSE); \
320 }
321
322 #endif
323
324 #if defined GMX_MM256_HERE && !defined GMX_DOUBLE
325
326 #define load_table_f(tab_coul_FDV0, ti_SSE, ti, ctab0_SSE, ctab1_SSE) \
327 { \
328 __m128 ctab_SSE[8],ctabt_SSE[4]; \
329 int j; \
330 \
331 /* Bit shifting would be faster, but AVX doesn't support that */ \
332 _mm256_store_si256((__m256i *)ti,ti_SSE); \
333 for(j=0; j<8; j++) \
334 { \
335 ctab_SSE[j] = _mm_load_ps(tab_coul_FDV0+ti[j]*4); \
336 } \
337 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(ctab_SSE[0],ctab_SSE[1],ctab_SSE[2],ctab_SSE[3],ctabt_SSE[0],ctabt_SSE[2]); \
338 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(ctab_SSE[4],ctab_SSE[5],ctab_SSE[6],ctab_SSE[7],ctabt_SSE[1],ctabt_SSE[3]); \
339 \
340 GMX_2_MM_TO_M256(ctabt_SSE[0],ctabt_SSE[1],ctab0_SSE); \
341 GMX_2_MM_TO_M256(ctabt_SSE[2],ctabt_SSE[3],ctab1_SSE); \
342 }
343
344 #define load_table_f_v(tab_coul_FDV0, ti_SSE, ti, ctab0_SSE, ctab1_SSE, ctabv_SSE) \
345 { \
346 __m128 ctab_SSE[8],ctabt_SSE[4],ctabvt_SSE[2]; \
347 int j; \
348 \
349 /* Bit shifting would be faster, but AVX doesn't support that */ \
350 _mm256_store_si256((__m256i *)ti,ti_SSE); \
351 for(j=0; j<8; j++) \
352 { \
353 ctab_SSE[j] = _mm_load_ps(tab_coul_FDV0+ti[j]*4); \
354 } \
355 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(ctab_SSE[0],ctab_SSE[1],ctab_SSE[2],ctab_SSE[3],ctabt_SSE[0],ctabt_SSE[2]); \
356 GMX_MM_SHUFFLE_4_PS_FIL01_TO_2_PS(ctab_SSE[4],ctab_SSE[5],ctab_SSE[6],ctab_SSE[7],ctabt_SSE[1],ctabt_SSE[3]); \
357 \
358 GMX_2_MM_TO_M256(ctabt_SSE[0],ctabt_SSE[1],ctab0_SSE); \
359 GMX_2_MM_TO_M256(ctabt_SSE[2],ctabt_SSE[3],ctab1_SSE); \
360 \
361 GMX_MM_SHUFFLE_4_PS_FIL2_TO_1_PS(ctab_SSE[0],ctab_SSE[1],ctab_SSE[2],ctab_SSE[3],ctabvt_SSE[0]); \
362 GMX_MM_SHUFFLE_4_PS_FIL2_TO_1_PS(ctab_SSE[4],ctab_SSE[5],ctab_SSE[6],ctab_SSE[7],ctabvt_SSE[1]); \
363 \
364 GMX_2_MM_TO_M256(ctabvt_SSE[0],ctabvt_SSE[1],ctabv_SSE); \
365 }
366
367 #endif
368
369 #if defined GMX_MM128_HERE && defined GMX_DOUBLE
370
371 #define load_table_f(tab_coul_F, ti_SSE, ti, ctab0_SSE, ctab1_SSE) \
372 { \
373 int idx[2]; \
374 __m128d ctab_SSE[2]; \
375 \
376 /* Without SSE4.1 the extract macro needs an immediate: unroll */ \
377 idx[0] = gmx_mm_extract_epi32(ti_SSE,0); \
378 ctab_SSE[0] = _mm_loadu_pd(tab_coul_F+idx[0]); \
379 idx[1] = gmx_mm_extract_epi32(ti_SSE,1); \
380 ctab_SSE[1] = _mm_loadu_pd(tab_coul_F+idx[1]); \
381 \
382 /* Shuffle the force table entries to a convenient order */ \
383 GMX_MM_TRANSPOSE2_OP_PD(ctab_SSE[0],ctab_SSE[1],ctab0_SSE,ctab1_SSE); \
384 /* The second force table entry should contain the difference */ \
385 ctab1_SSE = _mm_sub_pd(ctab1_SSE,ctab0_SSE); \
386 }
387
388 #define load_table_f_v(tab_coul_F, tab_coul_V, ti_SSE, ti, ctab0_SSE, ctab1_SSE, ctabv_SSE) \
389 { \
390 int idx[2]; \
391 __m128d ctab_SSE[4]; \
392 \
393 /* Without SSE4.1 the extract macro needs an immediate: unroll */ \
394 idx[0] = gmx_mm_extract_epi32(ti_SSE,0); \
395 ctab_SSE[0] = _mm_loadu_pd(tab_coul_F+idx[0]); \
396 idx[1] = gmx_mm_extract_epi32(ti_SSE,1); \
397 ctab_SSE[1] = _mm_loadu_pd(tab_coul_F+idx[1]); \
398 \
399 /* Shuffle the force table entries to a convenient order */ \
400 GMX_MM_TRANSPOSE2_OP_PD(ctab_SSE[0],ctab_SSE[1],ctab0_SSE,ctab1_SSE); \
401 /* The second force table entry should contain the difference */ \
402 ctab1_SSE = _mm_sub_pd(ctab1_SSE,ctab0_SSE); \
403 \
404 ctab_SSE[2] = _mm_loadu_pd(tab_coul_V+idx[0]); \
405 ctab_SSE[3] = _mm_loadu_pd(tab_coul_V+idx[1]); \
406 \
407 /* Shuffle the energy table entries to a single register */ \
408 ctabv_SSE = _mm_shuffle_pd(ctab_SSE[2],ctab_SSE[3],_MM_SHUFFLE2(0,0)); \
409 }
410
411 #endif
412
413 #if defined GMX_MM256_HERE && defined GMX_DOUBLE
414
415 /* Put two 128-bit 2-double registers into one 256-bit 4-ouble register */
416 #define GMX_2_M128D_TO_M256D(in0,in1,out) \
417 { \
418 out = _mm256_insertf128_pd(_mm256_castpd128_pd256(in0),in1,1); \
419 }
420
421 #define load_table_f(tab_coul_F, ti_SSE, ti, ctab0_SSE, ctab1_SSE) \
422 { \
423 __m128d ctab_SSE[4],tr_SSE[4]; \
424 int j; \
425 \
426 _mm_store_si128((__m128i *)ti,ti_SSE); \
427 for(j=0; j<4; j++) \
428 { \
429 ctab_SSE[j] = _mm_loadu_pd(tab_coul_F+ti[j]); \
430 } \
431 /* Shuffle the force table entries to a convenient order */ \
432 GMX_MM_TRANSPOSE2_OP_PD(ctab_SSE[0],ctab_SSE[1],tr_SSE[0],tr_SSE[1]); \
433 GMX_MM_TRANSPOSE2_OP_PD(ctab_SSE[2],ctab_SSE[3],tr_SSE[2],tr_SSE[3]); \
434 GMX_2_M128D_TO_M256D(tr_SSE[0],tr_SSE[2],ctab0_SSE); \
435 GMX_2_M128D_TO_M256D(tr_SSE[1],tr_SSE[3],ctab1_SSE); \
436 /* The second force table entry should contain the difference */ \
437 ctab1_SSE = _mm256_sub_pd(ctab1_SSE,ctab0_SSE); \
438 }
439
440 #define load_table_f_v(tab_coul_F, tab_coul_V, ti_SSE, ti, ctab0_SSE, ctab1_SSE, ctabv_SSE) \
441 { \
442 __m128d ctab_SSE[8],tr_SSE[4]; \
443 int j; \
444 \
445 _mm_store_si128((__m128i *)ti,ti_SSE); \
446 for(j=0; j<4; j++) \
447 { \
448 ctab_SSE[j] = _mm_loadu_pd(tab_coul_F+ti[j]); \
449 } \
450 /* Shuffle the force table entries to a convenient order */ \
451 GMX_MM_TRANSPOSE2_OP_PD(ctab_SSE[0],ctab_SSE[1],tr_SSE[0],tr_SSE[1]); \
452 GMX_MM_TRANSPOSE2_OP_PD(ctab_SSE[2],ctab_SSE[3],tr_SSE[2],tr_SSE[3]); \
453 GMX_2_M128D_TO_M256D(tr_SSE[0],tr_SSE[2],ctab0_SSE); \
454 GMX_2_M128D_TO_M256D(tr_SSE[1],tr_SSE[3],ctab1_SSE); \
455 /* The second force table entry should contain the difference */ \
456 ctab1_SSE = _mm256_sub_pd(ctab1_SSE,ctab0_SSE); \
457 \
458 for(j=0; j<4; j++) \
459 { \
460 ctab_SSE[4+j] = _mm_loadu_pd(tab_coul_V+ti[j]); \
461 } \
462 /* Shuffle the energy table entries to a single register */ \
463 GMX_2_M128D_TO_M256D(_mm_shuffle_pd(ctab_SSE[4],ctab_SSE[5],_MM_SHUFFLE2(0,0)),_mm_shuffle_pd(ctab_SSE[6],ctab_SSE[7],_MM_SHUFFLE2(0,0)),ctabv_SSE); \
464 }
465
466 #endif
467
468
469 /* Add energy register to possibly multiple terms in the energy array.
470 * This function is the same for SSE/AVX single/double.
471 */
472 static inline void add_ener_grp(gmx_mm_pr e_SSE,real *v,int *offset_jj)
473 {
474 int jj;
475
476 /* We need to balance the number of store operations with
477 * the rapidly increases number of combinations of energy groups.
478 * We add to a temporary buffer for 1 i-group vs 2 j-groups.
479 */
480 for(jj=0; jj<(UNROLLJ>>1); jj++)
481 {
482 gmx_mm_pr v_SSE;
483
484 v_SSE = gmx_load_pr(v+offset_jj[jj]+jj*UNROLLJ);
485 gmx_store_pr(v+offset_jj[jj]+jj*UNROLLJ,gmx_add_pr(v_SSE,e_SSE));
486 }
487 }
488
489 #endif /* _nbnxn_kernel_sse_utils_h_ */
The ultimate molecular dynamics simulation package