2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014,2015,2017, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_256_single kernel generator.
44 #include "../nb_kernel.h"
45 #include "gromacs/gmxlib/nrnb.h"
47 #include "kernelutil_x86_avx_256_single.h"
50 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_VF_avx_256_single
51 * Electrostatics interaction: ReactionField
52 * VdW interaction: CubicSplineTable
53 * Geometry: Water4-Particle
54 * Calculate force/pot: PotentialAndForce
57 nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_VF_avx_256_single
58 (t_nblist
* gmx_restrict nlist
,
59 rvec
* gmx_restrict xx
,
60 rvec
* gmx_restrict ff
,
61 struct t_forcerec
* gmx_restrict fr
,
62 t_mdatoms
* gmx_restrict mdatoms
,
63 nb_kernel_data_t gmx_unused
* gmx_restrict kernel_data
,
64 t_nrnb
* gmx_restrict nrnb
)
66 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
67 * just 0 for non-waters.
68 * Suffixes A,B,C,D,E,F,G,H refer to j loop unrolling done with AVX, e.g. for the eight different
69 * jnr indices corresponding to data put in the four positions in the SIMD register.
71 int i_shift_offset
,i_coord_offset
,outeriter
,inneriter
;
72 int j_index_start
,j_index_end
,jidx
,nri
,inr
,ggid
,iidx
;
73 int jnrA
,jnrB
,jnrC
,jnrD
;
74 int jnrE
,jnrF
,jnrG
,jnrH
;
75 int jnrlistA
,jnrlistB
,jnrlistC
,jnrlistD
;
76 int jnrlistE
,jnrlistF
,jnrlistG
,jnrlistH
;
77 int j_coord_offsetA
,j_coord_offsetB
,j_coord_offsetC
,j_coord_offsetD
;
78 int j_coord_offsetE
,j_coord_offsetF
,j_coord_offsetG
,j_coord_offsetH
;
79 int *iinr
,*jindex
,*jjnr
,*shiftidx
,*gid
;
81 real
*shiftvec
,*fshift
,*x
,*f
;
82 real
*fjptrA
,*fjptrB
,*fjptrC
,*fjptrD
,*fjptrE
,*fjptrF
,*fjptrG
,*fjptrH
;
84 __m256 tx
,ty
,tz
,fscal
,rcutoff
,rcutoff2
,jidxall
;
85 real
* vdwioffsetptr0
;
86 __m256 ix0
,iy0
,iz0
,fix0
,fiy0
,fiz0
,iq0
,isai0
;
87 real
* vdwioffsetptr1
;
88 __m256 ix1
,iy1
,iz1
,fix1
,fiy1
,fiz1
,iq1
,isai1
;
89 real
* vdwioffsetptr2
;
90 __m256 ix2
,iy2
,iz2
,fix2
,fiy2
,fiz2
,iq2
,isai2
;
91 real
* vdwioffsetptr3
;
92 __m256 ix3
,iy3
,iz3
,fix3
,fiy3
,fiz3
,iq3
,isai3
;
93 int vdwjidx0A
,vdwjidx0B
,vdwjidx0C
,vdwjidx0D
,vdwjidx0E
,vdwjidx0F
,vdwjidx0G
,vdwjidx0H
;
94 __m256 jx0
,jy0
,jz0
,fjx0
,fjy0
,fjz0
,jq0
,isaj0
;
95 __m256 dx00
,dy00
,dz00
,rsq00
,rinv00
,rinvsq00
,r00
,qq00
,c6_00
,c12_00
;
96 __m256 dx10
,dy10
,dz10
,rsq10
,rinv10
,rinvsq10
,r10
,qq10
,c6_10
,c12_10
;
97 __m256 dx20
,dy20
,dz20
,rsq20
,rinv20
,rinvsq20
,r20
,qq20
,c6_20
,c12_20
;
98 __m256 dx30
,dy30
,dz30
,rsq30
,rinv30
,rinvsq30
,r30
,qq30
,c6_30
,c12_30
;
99 __m256 velec
,felec
,velecsum
,facel
,crf
,krf
,krf2
;
102 __m256 rinvsix
,rvdw
,vvdw
,vvdw6
,vvdw12
,fvdw
,fvdw6
,fvdw12
,vvdwsum
,sh_vdw_invrcut6
;
105 __m256 one_sixth
= _mm256_set1_ps(1.0/6.0);
106 __m256 one_twelfth
= _mm256_set1_ps(1.0/12.0);
108 __m128i vfitab_lo
,vfitab_hi
;
109 __m128i ifour
= _mm_set1_epi32(4);
110 __m256 rt
,vfeps
,vftabscale
,Y
,F
,G
,H
,Heps
,Fp
,VV
,FF
;
112 __m256 dummy_mask
,cutoff_mask
;
113 __m256 signbit
= _mm256_castsi256_ps( _mm256_set1_epi32(0x80000000) );
114 __m256 one
= _mm256_set1_ps(1.0);
115 __m256 two
= _mm256_set1_ps(2.0);
121 jindex
= nlist
->jindex
;
123 shiftidx
= nlist
->shift
;
125 shiftvec
= fr
->shift_vec
[0];
126 fshift
= fr
->fshift
[0];
127 facel
= _mm256_set1_ps(fr
->ic
->epsfac
);
128 charge
= mdatoms
->chargeA
;
129 krf
= _mm256_set1_ps(fr
->ic
->k_rf
);
130 krf2
= _mm256_set1_ps(fr
->ic
->k_rf
*2.0);
131 crf
= _mm256_set1_ps(fr
->ic
->c_rf
);
132 nvdwtype
= fr
->ntype
;
134 vdwtype
= mdatoms
->typeA
;
136 vftab
= kernel_data
->table_vdw
->data
;
137 vftabscale
= _mm256_set1_ps(kernel_data
->table_vdw
->scale
);
139 /* Setup water-specific parameters */
140 inr
= nlist
->iinr
[0];
141 iq1
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+1]));
142 iq2
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+2]));
143 iq3
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+3]));
144 vdwioffsetptr0
= vdwparam
+2*nvdwtype
*vdwtype
[inr
+0];
146 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
147 rcutoff_scalar
= fr
->ic
->rcoulomb
;
148 rcutoff
= _mm256_set1_ps(rcutoff_scalar
);
149 rcutoff2
= _mm256_mul_ps(rcutoff
,rcutoff
);
151 /* Avoid stupid compiler warnings */
152 jnrA
= jnrB
= jnrC
= jnrD
= jnrE
= jnrF
= jnrG
= jnrH
= 0;
165 for(iidx
=0;iidx
<4*DIM
;iidx
++)
170 /* Start outer loop over neighborlists */
171 for(iidx
=0; iidx
<nri
; iidx
++)
173 /* Load shift vector for this list */
174 i_shift_offset
= DIM
*shiftidx
[iidx
];
176 /* Load limits for loop over neighbors */
177 j_index_start
= jindex
[iidx
];
178 j_index_end
= jindex
[iidx
+1];
180 /* Get outer coordinate index */
182 i_coord_offset
= DIM
*inr
;
184 /* Load i particle coords and add shift vector */
185 gmx_mm256_load_shift_and_4rvec_broadcast_ps(shiftvec
+i_shift_offset
,x
+i_coord_offset
,
186 &ix0
,&iy0
,&iz0
,&ix1
,&iy1
,&iz1
,&ix2
,&iy2
,&iz2
,&ix3
,&iy3
,&iz3
);
188 fix0
= _mm256_setzero_ps();
189 fiy0
= _mm256_setzero_ps();
190 fiz0
= _mm256_setzero_ps();
191 fix1
= _mm256_setzero_ps();
192 fiy1
= _mm256_setzero_ps();
193 fiz1
= _mm256_setzero_ps();
194 fix2
= _mm256_setzero_ps();
195 fiy2
= _mm256_setzero_ps();
196 fiz2
= _mm256_setzero_ps();
197 fix3
= _mm256_setzero_ps();
198 fiy3
= _mm256_setzero_ps();
199 fiz3
= _mm256_setzero_ps();
201 /* Reset potential sums */
202 velecsum
= _mm256_setzero_ps();
203 vvdwsum
= _mm256_setzero_ps();
205 /* Start inner kernel loop */
206 for(jidx
=j_index_start
; jidx
<j_index_end
&& jjnr
[jidx
+7]>=0; jidx
+=8)
209 /* Get j neighbor index, and coordinate index */
218 j_coord_offsetA
= DIM
*jnrA
;
219 j_coord_offsetB
= DIM
*jnrB
;
220 j_coord_offsetC
= DIM
*jnrC
;
221 j_coord_offsetD
= DIM
*jnrD
;
222 j_coord_offsetE
= DIM
*jnrE
;
223 j_coord_offsetF
= DIM
*jnrF
;
224 j_coord_offsetG
= DIM
*jnrG
;
225 j_coord_offsetH
= DIM
*jnrH
;
227 /* load j atom coordinates */
228 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
229 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
230 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
231 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
234 /* Calculate displacement vector */
235 dx00
= _mm256_sub_ps(ix0
,jx0
);
236 dy00
= _mm256_sub_ps(iy0
,jy0
);
237 dz00
= _mm256_sub_ps(iz0
,jz0
);
238 dx10
= _mm256_sub_ps(ix1
,jx0
);
239 dy10
= _mm256_sub_ps(iy1
,jy0
);
240 dz10
= _mm256_sub_ps(iz1
,jz0
);
241 dx20
= _mm256_sub_ps(ix2
,jx0
);
242 dy20
= _mm256_sub_ps(iy2
,jy0
);
243 dz20
= _mm256_sub_ps(iz2
,jz0
);
244 dx30
= _mm256_sub_ps(ix3
,jx0
);
245 dy30
= _mm256_sub_ps(iy3
,jy0
);
246 dz30
= _mm256_sub_ps(iz3
,jz0
);
248 /* Calculate squared distance and things based on it */
249 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
250 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
251 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
252 rsq30
= gmx_mm256_calc_rsq_ps(dx30
,dy30
,dz30
);
254 rinv00
= avx256_invsqrt_f(rsq00
);
255 rinv10
= avx256_invsqrt_f(rsq10
);
256 rinv20
= avx256_invsqrt_f(rsq20
);
257 rinv30
= avx256_invsqrt_f(rsq30
);
259 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
260 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
261 rinvsq30
= _mm256_mul_ps(rinv30
,rinv30
);
263 /* Load parameters for j particles */
264 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
265 charge
+jnrC
+0,charge
+jnrD
+0,
266 charge
+jnrE
+0,charge
+jnrF
+0,
267 charge
+jnrG
+0,charge
+jnrH
+0);
268 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
269 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
270 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
271 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
272 vdwjidx0E
= 2*vdwtype
[jnrE
+0];
273 vdwjidx0F
= 2*vdwtype
[jnrF
+0];
274 vdwjidx0G
= 2*vdwtype
[jnrG
+0];
275 vdwjidx0H
= 2*vdwtype
[jnrH
+0];
277 fjx0
= _mm256_setzero_ps();
278 fjy0
= _mm256_setzero_ps();
279 fjz0
= _mm256_setzero_ps();
281 /**************************
282 * CALCULATE INTERACTIONS *
283 **************************/
285 r00
= _mm256_mul_ps(rsq00
,rinv00
);
287 /* Compute parameters for interactions between i and j atoms */
288 gmx_mm256_load_8pair_swizzle_ps(vdwioffsetptr0
+vdwjidx0A
,
289 vdwioffsetptr0
+vdwjidx0B
,
290 vdwioffsetptr0
+vdwjidx0C
,
291 vdwioffsetptr0
+vdwjidx0D
,
292 vdwioffsetptr0
+vdwjidx0E
,
293 vdwioffsetptr0
+vdwjidx0F
,
294 vdwioffsetptr0
+vdwjidx0G
,
295 vdwioffsetptr0
+vdwjidx0H
,
298 /* Calculate table index by multiplying r with table scale and truncate to integer */
299 rt
= _mm256_mul_ps(r00
,vftabscale
);
300 vfitab
= _mm256_cvttps_epi32(rt
);
301 vfeps
= _mm256_sub_ps(rt
,_mm256_round_ps(rt
, _MM_FROUND_FLOOR
));
302 /* AVX1 does not support 256-bit integer operations, so now we go to 128-bit mode... */
303 vfitab_lo
= _mm256_extractf128_si256(vfitab
,0x0);
304 vfitab_hi
= _mm256_extractf128_si256(vfitab
,0x1);
305 vfitab_lo
= _mm_slli_epi32(vfitab_lo
,3);
306 vfitab_hi
= _mm_slli_epi32(vfitab_hi
,3);
308 /* CUBIC SPLINE TABLE DISPERSION */
309 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
310 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
311 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
312 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
313 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
314 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
315 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
316 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
317 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
318 Heps
= _mm256_mul_ps(vfeps
,H
);
319 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
320 VV
= _mm256_add_ps(Y
,_mm256_mul_ps(vfeps
,Fp
));
321 vvdw6
= _mm256_mul_ps(c6_00
,VV
);
322 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
323 fvdw6
= _mm256_mul_ps(c6_00
,FF
);
325 /* CUBIC SPLINE TABLE REPULSION */
326 vfitab_lo
= _mm_add_epi32(vfitab_lo
,ifour
);
327 vfitab_hi
= _mm_add_epi32(vfitab_hi
,ifour
);
328 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
329 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
330 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
331 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
332 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
333 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
334 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
335 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
336 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
337 Heps
= _mm256_mul_ps(vfeps
,H
);
338 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
339 VV
= _mm256_add_ps(Y
,_mm256_mul_ps(vfeps
,Fp
));
340 vvdw12
= _mm256_mul_ps(c12_00
,VV
);
341 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
342 fvdw12
= _mm256_mul_ps(c12_00
,FF
);
343 vvdw
= _mm256_add_ps(vvdw12
,vvdw6
);
344 fvdw
= _mm256_xor_ps(signbit
,_mm256_mul_ps(_mm256_add_ps(fvdw6
,fvdw12
),_mm256_mul_ps(vftabscale
,rinv00
)));
346 /* Update potential sum for this i atom from the interaction with this j atom. */
347 vvdwsum
= _mm256_add_ps(vvdwsum
,vvdw
);
351 /* Calculate temporary vectorial force */
352 tx
= _mm256_mul_ps(fscal
,dx00
);
353 ty
= _mm256_mul_ps(fscal
,dy00
);
354 tz
= _mm256_mul_ps(fscal
,dz00
);
356 /* Update vectorial force */
357 fix0
= _mm256_add_ps(fix0
,tx
);
358 fiy0
= _mm256_add_ps(fiy0
,ty
);
359 fiz0
= _mm256_add_ps(fiz0
,tz
);
361 fjx0
= _mm256_add_ps(fjx0
,tx
);
362 fjy0
= _mm256_add_ps(fjy0
,ty
);
363 fjz0
= _mm256_add_ps(fjz0
,tz
);
365 /**************************
366 * CALCULATE INTERACTIONS *
367 **************************/
369 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
372 /* Compute parameters for interactions between i and j atoms */
373 qq10
= _mm256_mul_ps(iq1
,jq0
);
375 /* REACTION-FIELD ELECTROSTATICS */
376 velec
= _mm256_mul_ps(qq10
,_mm256_sub_ps(_mm256_add_ps(rinv10
,_mm256_mul_ps(krf
,rsq10
)),crf
));
377 felec
= _mm256_mul_ps(qq10
,_mm256_sub_ps(_mm256_mul_ps(rinv10
,rinvsq10
),krf2
));
379 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
381 /* Update potential sum for this i atom from the interaction with this j atom. */
382 velec
= _mm256_and_ps(velec
,cutoff_mask
);
383 velecsum
= _mm256_add_ps(velecsum
,velec
);
387 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
389 /* Calculate temporary vectorial force */
390 tx
= _mm256_mul_ps(fscal
,dx10
);
391 ty
= _mm256_mul_ps(fscal
,dy10
);
392 tz
= _mm256_mul_ps(fscal
,dz10
);
394 /* Update vectorial force */
395 fix1
= _mm256_add_ps(fix1
,tx
);
396 fiy1
= _mm256_add_ps(fiy1
,ty
);
397 fiz1
= _mm256_add_ps(fiz1
,tz
);
399 fjx0
= _mm256_add_ps(fjx0
,tx
);
400 fjy0
= _mm256_add_ps(fjy0
,ty
);
401 fjz0
= _mm256_add_ps(fjz0
,tz
);
405 /**************************
406 * CALCULATE INTERACTIONS *
407 **************************/
409 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
412 /* Compute parameters for interactions between i and j atoms */
413 qq20
= _mm256_mul_ps(iq2
,jq0
);
415 /* REACTION-FIELD ELECTROSTATICS */
416 velec
= _mm256_mul_ps(qq20
,_mm256_sub_ps(_mm256_add_ps(rinv20
,_mm256_mul_ps(krf
,rsq20
)),crf
));
417 felec
= _mm256_mul_ps(qq20
,_mm256_sub_ps(_mm256_mul_ps(rinv20
,rinvsq20
),krf2
));
419 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
421 /* Update potential sum for this i atom from the interaction with this j atom. */
422 velec
= _mm256_and_ps(velec
,cutoff_mask
);
423 velecsum
= _mm256_add_ps(velecsum
,velec
);
427 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
429 /* Calculate temporary vectorial force */
430 tx
= _mm256_mul_ps(fscal
,dx20
);
431 ty
= _mm256_mul_ps(fscal
,dy20
);
432 tz
= _mm256_mul_ps(fscal
,dz20
);
434 /* Update vectorial force */
435 fix2
= _mm256_add_ps(fix2
,tx
);
436 fiy2
= _mm256_add_ps(fiy2
,ty
);
437 fiz2
= _mm256_add_ps(fiz2
,tz
);
439 fjx0
= _mm256_add_ps(fjx0
,tx
);
440 fjy0
= _mm256_add_ps(fjy0
,ty
);
441 fjz0
= _mm256_add_ps(fjz0
,tz
);
445 /**************************
446 * CALCULATE INTERACTIONS *
447 **************************/
449 if (gmx_mm256_any_lt(rsq30
,rcutoff2
))
452 /* Compute parameters for interactions between i and j atoms */
453 qq30
= _mm256_mul_ps(iq3
,jq0
);
455 /* REACTION-FIELD ELECTROSTATICS */
456 velec
= _mm256_mul_ps(qq30
,_mm256_sub_ps(_mm256_add_ps(rinv30
,_mm256_mul_ps(krf
,rsq30
)),crf
));
457 felec
= _mm256_mul_ps(qq30
,_mm256_sub_ps(_mm256_mul_ps(rinv30
,rinvsq30
),krf2
));
459 cutoff_mask
= _mm256_cmp_ps(rsq30
,rcutoff2
,_CMP_LT_OQ
);
461 /* Update potential sum for this i atom from the interaction with this j atom. */
462 velec
= _mm256_and_ps(velec
,cutoff_mask
);
463 velecsum
= _mm256_add_ps(velecsum
,velec
);
467 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
469 /* Calculate temporary vectorial force */
470 tx
= _mm256_mul_ps(fscal
,dx30
);
471 ty
= _mm256_mul_ps(fscal
,dy30
);
472 tz
= _mm256_mul_ps(fscal
,dz30
);
474 /* Update vectorial force */
475 fix3
= _mm256_add_ps(fix3
,tx
);
476 fiy3
= _mm256_add_ps(fiy3
,ty
);
477 fiz3
= _mm256_add_ps(fiz3
,tz
);
479 fjx0
= _mm256_add_ps(fjx0
,tx
);
480 fjy0
= _mm256_add_ps(fjy0
,ty
);
481 fjz0
= _mm256_add_ps(fjz0
,tz
);
485 fjptrA
= f
+j_coord_offsetA
;
486 fjptrB
= f
+j_coord_offsetB
;
487 fjptrC
= f
+j_coord_offsetC
;
488 fjptrD
= f
+j_coord_offsetD
;
489 fjptrE
= f
+j_coord_offsetE
;
490 fjptrF
= f
+j_coord_offsetF
;
491 fjptrG
= f
+j_coord_offsetG
;
492 fjptrH
= f
+j_coord_offsetH
;
494 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
496 /* Inner loop uses 167 flops */
502 /* Get j neighbor index, and coordinate index */
503 jnrlistA
= jjnr
[jidx
];
504 jnrlistB
= jjnr
[jidx
+1];
505 jnrlistC
= jjnr
[jidx
+2];
506 jnrlistD
= jjnr
[jidx
+3];
507 jnrlistE
= jjnr
[jidx
+4];
508 jnrlistF
= jjnr
[jidx
+5];
509 jnrlistG
= jjnr
[jidx
+6];
510 jnrlistH
= jjnr
[jidx
+7];
511 /* Sign of each element will be negative for non-real atoms.
512 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
513 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
515 dummy_mask
= gmx_mm256_set_m128(gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
+4)),_mm_setzero_si128())),
516 gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
)),_mm_setzero_si128())));
518 jnrA
= (jnrlistA
>=0) ? jnrlistA
: 0;
519 jnrB
= (jnrlistB
>=0) ? jnrlistB
: 0;
520 jnrC
= (jnrlistC
>=0) ? jnrlistC
: 0;
521 jnrD
= (jnrlistD
>=0) ? jnrlistD
: 0;
522 jnrE
= (jnrlistE
>=0) ? jnrlistE
: 0;
523 jnrF
= (jnrlistF
>=0) ? jnrlistF
: 0;
524 jnrG
= (jnrlistG
>=0) ? jnrlistG
: 0;
525 jnrH
= (jnrlistH
>=0) ? jnrlistH
: 0;
526 j_coord_offsetA
= DIM
*jnrA
;
527 j_coord_offsetB
= DIM
*jnrB
;
528 j_coord_offsetC
= DIM
*jnrC
;
529 j_coord_offsetD
= DIM
*jnrD
;
530 j_coord_offsetE
= DIM
*jnrE
;
531 j_coord_offsetF
= DIM
*jnrF
;
532 j_coord_offsetG
= DIM
*jnrG
;
533 j_coord_offsetH
= DIM
*jnrH
;
535 /* load j atom coordinates */
536 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
537 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
538 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
539 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
542 /* Calculate displacement vector */
543 dx00
= _mm256_sub_ps(ix0
,jx0
);
544 dy00
= _mm256_sub_ps(iy0
,jy0
);
545 dz00
= _mm256_sub_ps(iz0
,jz0
);
546 dx10
= _mm256_sub_ps(ix1
,jx0
);
547 dy10
= _mm256_sub_ps(iy1
,jy0
);
548 dz10
= _mm256_sub_ps(iz1
,jz0
);
549 dx20
= _mm256_sub_ps(ix2
,jx0
);
550 dy20
= _mm256_sub_ps(iy2
,jy0
);
551 dz20
= _mm256_sub_ps(iz2
,jz0
);
552 dx30
= _mm256_sub_ps(ix3
,jx0
);
553 dy30
= _mm256_sub_ps(iy3
,jy0
);
554 dz30
= _mm256_sub_ps(iz3
,jz0
);
556 /* Calculate squared distance and things based on it */
557 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
558 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
559 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
560 rsq30
= gmx_mm256_calc_rsq_ps(dx30
,dy30
,dz30
);
562 rinv00
= avx256_invsqrt_f(rsq00
);
563 rinv10
= avx256_invsqrt_f(rsq10
);
564 rinv20
= avx256_invsqrt_f(rsq20
);
565 rinv30
= avx256_invsqrt_f(rsq30
);
567 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
568 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
569 rinvsq30
= _mm256_mul_ps(rinv30
,rinv30
);
571 /* Load parameters for j particles */
572 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
573 charge
+jnrC
+0,charge
+jnrD
+0,
574 charge
+jnrE
+0,charge
+jnrF
+0,
575 charge
+jnrG
+0,charge
+jnrH
+0);
576 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
577 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
578 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
579 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
580 vdwjidx0E
= 2*vdwtype
[jnrE
+0];
581 vdwjidx0F
= 2*vdwtype
[jnrF
+0];
582 vdwjidx0G
= 2*vdwtype
[jnrG
+0];
583 vdwjidx0H
= 2*vdwtype
[jnrH
+0];
585 fjx0
= _mm256_setzero_ps();
586 fjy0
= _mm256_setzero_ps();
587 fjz0
= _mm256_setzero_ps();
589 /**************************
590 * CALCULATE INTERACTIONS *
591 **************************/
593 r00
= _mm256_mul_ps(rsq00
,rinv00
);
594 r00
= _mm256_andnot_ps(dummy_mask
,r00
);
596 /* Compute parameters for interactions between i and j atoms */
597 gmx_mm256_load_8pair_swizzle_ps(vdwioffsetptr0
+vdwjidx0A
,
598 vdwioffsetptr0
+vdwjidx0B
,
599 vdwioffsetptr0
+vdwjidx0C
,
600 vdwioffsetptr0
+vdwjidx0D
,
601 vdwioffsetptr0
+vdwjidx0E
,
602 vdwioffsetptr0
+vdwjidx0F
,
603 vdwioffsetptr0
+vdwjidx0G
,
604 vdwioffsetptr0
+vdwjidx0H
,
607 /* Calculate table index by multiplying r with table scale and truncate to integer */
608 rt
= _mm256_mul_ps(r00
,vftabscale
);
609 vfitab
= _mm256_cvttps_epi32(rt
);
610 vfeps
= _mm256_sub_ps(rt
,_mm256_round_ps(rt
, _MM_FROUND_FLOOR
));
611 /* AVX1 does not support 256-bit integer operations, so now we go to 128-bit mode... */
612 vfitab_lo
= _mm256_extractf128_si256(vfitab
,0x0);
613 vfitab_hi
= _mm256_extractf128_si256(vfitab
,0x1);
614 vfitab_lo
= _mm_slli_epi32(vfitab_lo
,3);
615 vfitab_hi
= _mm_slli_epi32(vfitab_hi
,3);
617 /* CUBIC SPLINE TABLE DISPERSION */
618 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
619 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
620 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
621 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
622 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
623 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
624 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
625 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
626 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
627 Heps
= _mm256_mul_ps(vfeps
,H
);
628 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
629 VV
= _mm256_add_ps(Y
,_mm256_mul_ps(vfeps
,Fp
));
630 vvdw6
= _mm256_mul_ps(c6_00
,VV
);
631 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
632 fvdw6
= _mm256_mul_ps(c6_00
,FF
);
634 /* CUBIC SPLINE TABLE REPULSION */
635 vfitab_lo
= _mm_add_epi32(vfitab_lo
,ifour
);
636 vfitab_hi
= _mm_add_epi32(vfitab_hi
,ifour
);
637 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
638 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
639 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
640 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
641 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
642 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
643 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
644 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
645 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
646 Heps
= _mm256_mul_ps(vfeps
,H
);
647 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
648 VV
= _mm256_add_ps(Y
,_mm256_mul_ps(vfeps
,Fp
));
649 vvdw12
= _mm256_mul_ps(c12_00
,VV
);
650 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
651 fvdw12
= _mm256_mul_ps(c12_00
,FF
);
652 vvdw
= _mm256_add_ps(vvdw12
,vvdw6
);
653 fvdw
= _mm256_xor_ps(signbit
,_mm256_mul_ps(_mm256_add_ps(fvdw6
,fvdw12
),_mm256_mul_ps(vftabscale
,rinv00
)));
655 /* Update potential sum for this i atom from the interaction with this j atom. */
656 vvdw
= _mm256_andnot_ps(dummy_mask
,vvdw
);
657 vvdwsum
= _mm256_add_ps(vvdwsum
,vvdw
);
661 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
663 /* Calculate temporary vectorial force */
664 tx
= _mm256_mul_ps(fscal
,dx00
);
665 ty
= _mm256_mul_ps(fscal
,dy00
);
666 tz
= _mm256_mul_ps(fscal
,dz00
);
668 /* Update vectorial force */
669 fix0
= _mm256_add_ps(fix0
,tx
);
670 fiy0
= _mm256_add_ps(fiy0
,ty
);
671 fiz0
= _mm256_add_ps(fiz0
,tz
);
673 fjx0
= _mm256_add_ps(fjx0
,tx
);
674 fjy0
= _mm256_add_ps(fjy0
,ty
);
675 fjz0
= _mm256_add_ps(fjz0
,tz
);
677 /**************************
678 * CALCULATE INTERACTIONS *
679 **************************/
681 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
684 /* Compute parameters for interactions between i and j atoms */
685 qq10
= _mm256_mul_ps(iq1
,jq0
);
687 /* REACTION-FIELD ELECTROSTATICS */
688 velec
= _mm256_mul_ps(qq10
,_mm256_sub_ps(_mm256_add_ps(rinv10
,_mm256_mul_ps(krf
,rsq10
)),crf
));
689 felec
= _mm256_mul_ps(qq10
,_mm256_sub_ps(_mm256_mul_ps(rinv10
,rinvsq10
),krf2
));
691 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
693 /* Update potential sum for this i atom from the interaction with this j atom. */
694 velec
= _mm256_and_ps(velec
,cutoff_mask
);
695 velec
= _mm256_andnot_ps(dummy_mask
,velec
);
696 velecsum
= _mm256_add_ps(velecsum
,velec
);
700 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
702 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
704 /* Calculate temporary vectorial force */
705 tx
= _mm256_mul_ps(fscal
,dx10
);
706 ty
= _mm256_mul_ps(fscal
,dy10
);
707 tz
= _mm256_mul_ps(fscal
,dz10
);
709 /* Update vectorial force */
710 fix1
= _mm256_add_ps(fix1
,tx
);
711 fiy1
= _mm256_add_ps(fiy1
,ty
);
712 fiz1
= _mm256_add_ps(fiz1
,tz
);
714 fjx0
= _mm256_add_ps(fjx0
,tx
);
715 fjy0
= _mm256_add_ps(fjy0
,ty
);
716 fjz0
= _mm256_add_ps(fjz0
,tz
);
720 /**************************
721 * CALCULATE INTERACTIONS *
722 **************************/
724 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
727 /* Compute parameters for interactions between i and j atoms */
728 qq20
= _mm256_mul_ps(iq2
,jq0
);
730 /* REACTION-FIELD ELECTROSTATICS */
731 velec
= _mm256_mul_ps(qq20
,_mm256_sub_ps(_mm256_add_ps(rinv20
,_mm256_mul_ps(krf
,rsq20
)),crf
));
732 felec
= _mm256_mul_ps(qq20
,_mm256_sub_ps(_mm256_mul_ps(rinv20
,rinvsq20
),krf2
));
734 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
736 /* Update potential sum for this i atom from the interaction with this j atom. */
737 velec
= _mm256_and_ps(velec
,cutoff_mask
);
738 velec
= _mm256_andnot_ps(dummy_mask
,velec
);
739 velecsum
= _mm256_add_ps(velecsum
,velec
);
743 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
745 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
747 /* Calculate temporary vectorial force */
748 tx
= _mm256_mul_ps(fscal
,dx20
);
749 ty
= _mm256_mul_ps(fscal
,dy20
);
750 tz
= _mm256_mul_ps(fscal
,dz20
);
752 /* Update vectorial force */
753 fix2
= _mm256_add_ps(fix2
,tx
);
754 fiy2
= _mm256_add_ps(fiy2
,ty
);
755 fiz2
= _mm256_add_ps(fiz2
,tz
);
757 fjx0
= _mm256_add_ps(fjx0
,tx
);
758 fjy0
= _mm256_add_ps(fjy0
,ty
);
759 fjz0
= _mm256_add_ps(fjz0
,tz
);
763 /**************************
764 * CALCULATE INTERACTIONS *
765 **************************/
767 if (gmx_mm256_any_lt(rsq30
,rcutoff2
))
770 /* Compute parameters for interactions between i and j atoms */
771 qq30
= _mm256_mul_ps(iq3
,jq0
);
773 /* REACTION-FIELD ELECTROSTATICS */
774 velec
= _mm256_mul_ps(qq30
,_mm256_sub_ps(_mm256_add_ps(rinv30
,_mm256_mul_ps(krf
,rsq30
)),crf
));
775 felec
= _mm256_mul_ps(qq30
,_mm256_sub_ps(_mm256_mul_ps(rinv30
,rinvsq30
),krf2
));
777 cutoff_mask
= _mm256_cmp_ps(rsq30
,rcutoff2
,_CMP_LT_OQ
);
779 /* Update potential sum for this i atom from the interaction with this j atom. */
780 velec
= _mm256_and_ps(velec
,cutoff_mask
);
781 velec
= _mm256_andnot_ps(dummy_mask
,velec
);
782 velecsum
= _mm256_add_ps(velecsum
,velec
);
786 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
788 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
790 /* Calculate temporary vectorial force */
791 tx
= _mm256_mul_ps(fscal
,dx30
);
792 ty
= _mm256_mul_ps(fscal
,dy30
);
793 tz
= _mm256_mul_ps(fscal
,dz30
);
795 /* Update vectorial force */
796 fix3
= _mm256_add_ps(fix3
,tx
);
797 fiy3
= _mm256_add_ps(fiy3
,ty
);
798 fiz3
= _mm256_add_ps(fiz3
,tz
);
800 fjx0
= _mm256_add_ps(fjx0
,tx
);
801 fjy0
= _mm256_add_ps(fjy0
,ty
);
802 fjz0
= _mm256_add_ps(fjz0
,tz
);
806 fjptrA
= (jnrlistA
>=0) ? f
+j_coord_offsetA
: scratch
;
807 fjptrB
= (jnrlistB
>=0) ? f
+j_coord_offsetB
: scratch
;
808 fjptrC
= (jnrlistC
>=0) ? f
+j_coord_offsetC
: scratch
;
809 fjptrD
= (jnrlistD
>=0) ? f
+j_coord_offsetD
: scratch
;
810 fjptrE
= (jnrlistE
>=0) ? f
+j_coord_offsetE
: scratch
;
811 fjptrF
= (jnrlistF
>=0) ? f
+j_coord_offsetF
: scratch
;
812 fjptrG
= (jnrlistG
>=0) ? f
+j_coord_offsetG
: scratch
;
813 fjptrH
= (jnrlistH
>=0) ? f
+j_coord_offsetH
: scratch
;
815 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
817 /* Inner loop uses 168 flops */
820 /* End of innermost loop */
822 gmx_mm256_update_iforce_4atom_swizzle_ps(fix0
,fiy0
,fiz0
,fix1
,fiy1
,fiz1
,fix2
,fiy2
,fiz2
,fix3
,fiy3
,fiz3
,
823 f
+i_coord_offset
,fshift
+i_shift_offset
);
826 /* Update potential energies */
827 gmx_mm256_update_1pot_ps(velecsum
,kernel_data
->energygrp_elec
+ggid
);
828 gmx_mm256_update_1pot_ps(vvdwsum
,kernel_data
->energygrp_vdw
+ggid
);
830 /* Increment number of inner iterations */
831 inneriter
+= j_index_end
- j_index_start
;
833 /* Outer loop uses 26 flops */
836 /* Increment number of outer iterations */
839 /* Update outer/inner flops */
841 inc_nrnb(nrnb
,eNR_NBKERNEL_ELEC_VDW_W4_VF
,outeriter
*26 + inneriter
*168);
844 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_F_avx_256_single
845 * Electrostatics interaction: ReactionField
846 * VdW interaction: CubicSplineTable
847 * Geometry: Water4-Particle
848 * Calculate force/pot: Force
851 nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_F_avx_256_single
852 (t_nblist
* gmx_restrict nlist
,
853 rvec
* gmx_restrict xx
,
854 rvec
* gmx_restrict ff
,
855 struct t_forcerec
* gmx_restrict fr
,
856 t_mdatoms
* gmx_restrict mdatoms
,
857 nb_kernel_data_t gmx_unused
* gmx_restrict kernel_data
,
858 t_nrnb
* gmx_restrict nrnb
)
860 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
861 * just 0 for non-waters.
862 * Suffixes A,B,C,D,E,F,G,H refer to j loop unrolling done with AVX, e.g. for the eight different
863 * jnr indices corresponding to data put in the four positions in the SIMD register.
865 int i_shift_offset
,i_coord_offset
,outeriter
,inneriter
;
866 int j_index_start
,j_index_end
,jidx
,nri
,inr
,ggid
,iidx
;
867 int jnrA
,jnrB
,jnrC
,jnrD
;
868 int jnrE
,jnrF
,jnrG
,jnrH
;
869 int jnrlistA
,jnrlistB
,jnrlistC
,jnrlistD
;
870 int jnrlistE
,jnrlistF
,jnrlistG
,jnrlistH
;
871 int j_coord_offsetA
,j_coord_offsetB
,j_coord_offsetC
,j_coord_offsetD
;
872 int j_coord_offsetE
,j_coord_offsetF
,j_coord_offsetG
,j_coord_offsetH
;
873 int *iinr
,*jindex
,*jjnr
,*shiftidx
,*gid
;
875 real
*shiftvec
,*fshift
,*x
,*f
;
876 real
*fjptrA
,*fjptrB
,*fjptrC
,*fjptrD
,*fjptrE
,*fjptrF
,*fjptrG
,*fjptrH
;
878 __m256 tx
,ty
,tz
,fscal
,rcutoff
,rcutoff2
,jidxall
;
879 real
* vdwioffsetptr0
;
880 __m256 ix0
,iy0
,iz0
,fix0
,fiy0
,fiz0
,iq0
,isai0
;
881 real
* vdwioffsetptr1
;
882 __m256 ix1
,iy1
,iz1
,fix1
,fiy1
,fiz1
,iq1
,isai1
;
883 real
* vdwioffsetptr2
;
884 __m256 ix2
,iy2
,iz2
,fix2
,fiy2
,fiz2
,iq2
,isai2
;
885 real
* vdwioffsetptr3
;
886 __m256 ix3
,iy3
,iz3
,fix3
,fiy3
,fiz3
,iq3
,isai3
;
887 int vdwjidx0A
,vdwjidx0B
,vdwjidx0C
,vdwjidx0D
,vdwjidx0E
,vdwjidx0F
,vdwjidx0G
,vdwjidx0H
;
888 __m256 jx0
,jy0
,jz0
,fjx0
,fjy0
,fjz0
,jq0
,isaj0
;
889 __m256 dx00
,dy00
,dz00
,rsq00
,rinv00
,rinvsq00
,r00
,qq00
,c6_00
,c12_00
;
890 __m256 dx10
,dy10
,dz10
,rsq10
,rinv10
,rinvsq10
,r10
,qq10
,c6_10
,c12_10
;
891 __m256 dx20
,dy20
,dz20
,rsq20
,rinv20
,rinvsq20
,r20
,qq20
,c6_20
,c12_20
;
892 __m256 dx30
,dy30
,dz30
,rsq30
,rinv30
,rinvsq30
,r30
,qq30
,c6_30
,c12_30
;
893 __m256 velec
,felec
,velecsum
,facel
,crf
,krf
,krf2
;
896 __m256 rinvsix
,rvdw
,vvdw
,vvdw6
,vvdw12
,fvdw
,fvdw6
,fvdw12
,vvdwsum
,sh_vdw_invrcut6
;
899 __m256 one_sixth
= _mm256_set1_ps(1.0/6.0);
900 __m256 one_twelfth
= _mm256_set1_ps(1.0/12.0);
902 __m128i vfitab_lo
,vfitab_hi
;
903 __m128i ifour
= _mm_set1_epi32(4);
904 __m256 rt
,vfeps
,vftabscale
,Y
,F
,G
,H
,Heps
,Fp
,VV
,FF
;
906 __m256 dummy_mask
,cutoff_mask
;
907 __m256 signbit
= _mm256_castsi256_ps( _mm256_set1_epi32(0x80000000) );
908 __m256 one
= _mm256_set1_ps(1.0);
909 __m256 two
= _mm256_set1_ps(2.0);
915 jindex
= nlist
->jindex
;
917 shiftidx
= nlist
->shift
;
919 shiftvec
= fr
->shift_vec
[0];
920 fshift
= fr
->fshift
[0];
921 facel
= _mm256_set1_ps(fr
->ic
->epsfac
);
922 charge
= mdatoms
->chargeA
;
923 krf
= _mm256_set1_ps(fr
->ic
->k_rf
);
924 krf2
= _mm256_set1_ps(fr
->ic
->k_rf
*2.0);
925 crf
= _mm256_set1_ps(fr
->ic
->c_rf
);
926 nvdwtype
= fr
->ntype
;
928 vdwtype
= mdatoms
->typeA
;
930 vftab
= kernel_data
->table_vdw
->data
;
931 vftabscale
= _mm256_set1_ps(kernel_data
->table_vdw
->scale
);
933 /* Setup water-specific parameters */
934 inr
= nlist
->iinr
[0];
935 iq1
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+1]));
936 iq2
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+2]));
937 iq3
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+3]));
938 vdwioffsetptr0
= vdwparam
+2*nvdwtype
*vdwtype
[inr
+0];
940 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
941 rcutoff_scalar
= fr
->ic
->rcoulomb
;
942 rcutoff
= _mm256_set1_ps(rcutoff_scalar
);
943 rcutoff2
= _mm256_mul_ps(rcutoff
,rcutoff
);
945 /* Avoid stupid compiler warnings */
946 jnrA
= jnrB
= jnrC
= jnrD
= jnrE
= jnrF
= jnrG
= jnrH
= 0;
959 for(iidx
=0;iidx
<4*DIM
;iidx
++)
964 /* Start outer loop over neighborlists */
965 for(iidx
=0; iidx
<nri
; iidx
++)
967 /* Load shift vector for this list */
968 i_shift_offset
= DIM
*shiftidx
[iidx
];
970 /* Load limits for loop over neighbors */
971 j_index_start
= jindex
[iidx
];
972 j_index_end
= jindex
[iidx
+1];
974 /* Get outer coordinate index */
976 i_coord_offset
= DIM
*inr
;
978 /* Load i particle coords and add shift vector */
979 gmx_mm256_load_shift_and_4rvec_broadcast_ps(shiftvec
+i_shift_offset
,x
+i_coord_offset
,
980 &ix0
,&iy0
,&iz0
,&ix1
,&iy1
,&iz1
,&ix2
,&iy2
,&iz2
,&ix3
,&iy3
,&iz3
);
982 fix0
= _mm256_setzero_ps();
983 fiy0
= _mm256_setzero_ps();
984 fiz0
= _mm256_setzero_ps();
985 fix1
= _mm256_setzero_ps();
986 fiy1
= _mm256_setzero_ps();
987 fiz1
= _mm256_setzero_ps();
988 fix2
= _mm256_setzero_ps();
989 fiy2
= _mm256_setzero_ps();
990 fiz2
= _mm256_setzero_ps();
991 fix3
= _mm256_setzero_ps();
992 fiy3
= _mm256_setzero_ps();
993 fiz3
= _mm256_setzero_ps();
995 /* Start inner kernel loop */
996 for(jidx
=j_index_start
; jidx
<j_index_end
&& jjnr
[jidx
+7]>=0; jidx
+=8)
999 /* Get j neighbor index, and coordinate index */
1001 jnrB
= jjnr
[jidx
+1];
1002 jnrC
= jjnr
[jidx
+2];
1003 jnrD
= jjnr
[jidx
+3];
1004 jnrE
= jjnr
[jidx
+4];
1005 jnrF
= jjnr
[jidx
+5];
1006 jnrG
= jjnr
[jidx
+6];
1007 jnrH
= jjnr
[jidx
+7];
1008 j_coord_offsetA
= DIM
*jnrA
;
1009 j_coord_offsetB
= DIM
*jnrB
;
1010 j_coord_offsetC
= DIM
*jnrC
;
1011 j_coord_offsetD
= DIM
*jnrD
;
1012 j_coord_offsetE
= DIM
*jnrE
;
1013 j_coord_offsetF
= DIM
*jnrF
;
1014 j_coord_offsetG
= DIM
*jnrG
;
1015 j_coord_offsetH
= DIM
*jnrH
;
1017 /* load j atom coordinates */
1018 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
1019 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
1020 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
1021 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
1024 /* Calculate displacement vector */
1025 dx00
= _mm256_sub_ps(ix0
,jx0
);
1026 dy00
= _mm256_sub_ps(iy0
,jy0
);
1027 dz00
= _mm256_sub_ps(iz0
,jz0
);
1028 dx10
= _mm256_sub_ps(ix1
,jx0
);
1029 dy10
= _mm256_sub_ps(iy1
,jy0
);
1030 dz10
= _mm256_sub_ps(iz1
,jz0
);
1031 dx20
= _mm256_sub_ps(ix2
,jx0
);
1032 dy20
= _mm256_sub_ps(iy2
,jy0
);
1033 dz20
= _mm256_sub_ps(iz2
,jz0
);
1034 dx30
= _mm256_sub_ps(ix3
,jx0
);
1035 dy30
= _mm256_sub_ps(iy3
,jy0
);
1036 dz30
= _mm256_sub_ps(iz3
,jz0
);
1038 /* Calculate squared distance and things based on it */
1039 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
1040 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
1041 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
1042 rsq30
= gmx_mm256_calc_rsq_ps(dx30
,dy30
,dz30
);
1044 rinv00
= avx256_invsqrt_f(rsq00
);
1045 rinv10
= avx256_invsqrt_f(rsq10
);
1046 rinv20
= avx256_invsqrt_f(rsq20
);
1047 rinv30
= avx256_invsqrt_f(rsq30
);
1049 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
1050 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
1051 rinvsq30
= _mm256_mul_ps(rinv30
,rinv30
);
1053 /* Load parameters for j particles */
1054 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
1055 charge
+jnrC
+0,charge
+jnrD
+0,
1056 charge
+jnrE
+0,charge
+jnrF
+0,
1057 charge
+jnrG
+0,charge
+jnrH
+0);
1058 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
1059 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
1060 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
1061 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
1062 vdwjidx0E
= 2*vdwtype
[jnrE
+0];
1063 vdwjidx0F
= 2*vdwtype
[jnrF
+0];
1064 vdwjidx0G
= 2*vdwtype
[jnrG
+0];
1065 vdwjidx0H
= 2*vdwtype
[jnrH
+0];
1067 fjx0
= _mm256_setzero_ps();
1068 fjy0
= _mm256_setzero_ps();
1069 fjz0
= _mm256_setzero_ps();
1071 /**************************
1072 * CALCULATE INTERACTIONS *
1073 **************************/
1075 r00
= _mm256_mul_ps(rsq00
,rinv00
);
1077 /* Compute parameters for interactions between i and j atoms */
1078 gmx_mm256_load_8pair_swizzle_ps(vdwioffsetptr0
+vdwjidx0A
,
1079 vdwioffsetptr0
+vdwjidx0B
,
1080 vdwioffsetptr0
+vdwjidx0C
,
1081 vdwioffsetptr0
+vdwjidx0D
,
1082 vdwioffsetptr0
+vdwjidx0E
,
1083 vdwioffsetptr0
+vdwjidx0F
,
1084 vdwioffsetptr0
+vdwjidx0G
,
1085 vdwioffsetptr0
+vdwjidx0H
,
1088 /* Calculate table index by multiplying r with table scale and truncate to integer */
1089 rt
= _mm256_mul_ps(r00
,vftabscale
);
1090 vfitab
= _mm256_cvttps_epi32(rt
);
1091 vfeps
= _mm256_sub_ps(rt
,_mm256_round_ps(rt
, _MM_FROUND_FLOOR
));
1092 /* AVX1 does not support 256-bit integer operations, so now we go to 128-bit mode... */
1093 vfitab_lo
= _mm256_extractf128_si256(vfitab
,0x0);
1094 vfitab_hi
= _mm256_extractf128_si256(vfitab
,0x1);
1095 vfitab_lo
= _mm_slli_epi32(vfitab_lo
,3);
1096 vfitab_hi
= _mm_slli_epi32(vfitab_hi
,3);
1098 /* CUBIC SPLINE TABLE DISPERSION */
1099 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
1100 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
1101 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
1102 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
1103 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
1104 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
1105 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
1106 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
1107 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
1108 Heps
= _mm256_mul_ps(vfeps
,H
);
1109 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
1110 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
1111 fvdw6
= _mm256_mul_ps(c6_00
,FF
);
1113 /* CUBIC SPLINE TABLE REPULSION */
1114 vfitab_lo
= _mm_add_epi32(vfitab_lo
,ifour
);
1115 vfitab_hi
= _mm_add_epi32(vfitab_hi
,ifour
);
1116 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
1117 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
1118 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
1119 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
1120 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
1121 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
1122 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
1123 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
1124 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
1125 Heps
= _mm256_mul_ps(vfeps
,H
);
1126 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
1127 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
1128 fvdw12
= _mm256_mul_ps(c12_00
,FF
);
1129 fvdw
= _mm256_xor_ps(signbit
,_mm256_mul_ps(_mm256_add_ps(fvdw6
,fvdw12
),_mm256_mul_ps(vftabscale
,rinv00
)));
1133 /* Calculate temporary vectorial force */
1134 tx
= _mm256_mul_ps(fscal
,dx00
);
1135 ty
= _mm256_mul_ps(fscal
,dy00
);
1136 tz
= _mm256_mul_ps(fscal
,dz00
);
1138 /* Update vectorial force */
1139 fix0
= _mm256_add_ps(fix0
,tx
);
1140 fiy0
= _mm256_add_ps(fiy0
,ty
);
1141 fiz0
= _mm256_add_ps(fiz0
,tz
);
1143 fjx0
= _mm256_add_ps(fjx0
,tx
);
1144 fjy0
= _mm256_add_ps(fjy0
,ty
);
1145 fjz0
= _mm256_add_ps(fjz0
,tz
);
1147 /**************************
1148 * CALCULATE INTERACTIONS *
1149 **************************/
1151 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
1154 /* Compute parameters for interactions between i and j atoms */
1155 qq10
= _mm256_mul_ps(iq1
,jq0
);
1157 /* REACTION-FIELD ELECTROSTATICS */
1158 felec
= _mm256_mul_ps(qq10
,_mm256_sub_ps(_mm256_mul_ps(rinv10
,rinvsq10
),krf2
));
1160 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
1164 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1166 /* Calculate temporary vectorial force */
1167 tx
= _mm256_mul_ps(fscal
,dx10
);
1168 ty
= _mm256_mul_ps(fscal
,dy10
);
1169 tz
= _mm256_mul_ps(fscal
,dz10
);
1171 /* Update vectorial force */
1172 fix1
= _mm256_add_ps(fix1
,tx
);
1173 fiy1
= _mm256_add_ps(fiy1
,ty
);
1174 fiz1
= _mm256_add_ps(fiz1
,tz
);
1176 fjx0
= _mm256_add_ps(fjx0
,tx
);
1177 fjy0
= _mm256_add_ps(fjy0
,ty
);
1178 fjz0
= _mm256_add_ps(fjz0
,tz
);
1182 /**************************
1183 * CALCULATE INTERACTIONS *
1184 **************************/
1186 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
1189 /* Compute parameters for interactions between i and j atoms */
1190 qq20
= _mm256_mul_ps(iq2
,jq0
);
1192 /* REACTION-FIELD ELECTROSTATICS */
1193 felec
= _mm256_mul_ps(qq20
,_mm256_sub_ps(_mm256_mul_ps(rinv20
,rinvsq20
),krf2
));
1195 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
1199 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1201 /* Calculate temporary vectorial force */
1202 tx
= _mm256_mul_ps(fscal
,dx20
);
1203 ty
= _mm256_mul_ps(fscal
,dy20
);
1204 tz
= _mm256_mul_ps(fscal
,dz20
);
1206 /* Update vectorial force */
1207 fix2
= _mm256_add_ps(fix2
,tx
);
1208 fiy2
= _mm256_add_ps(fiy2
,ty
);
1209 fiz2
= _mm256_add_ps(fiz2
,tz
);
1211 fjx0
= _mm256_add_ps(fjx0
,tx
);
1212 fjy0
= _mm256_add_ps(fjy0
,ty
);
1213 fjz0
= _mm256_add_ps(fjz0
,tz
);
1217 /**************************
1218 * CALCULATE INTERACTIONS *
1219 **************************/
1221 if (gmx_mm256_any_lt(rsq30
,rcutoff2
))
1224 /* Compute parameters for interactions between i and j atoms */
1225 qq30
= _mm256_mul_ps(iq3
,jq0
);
1227 /* REACTION-FIELD ELECTROSTATICS */
1228 felec
= _mm256_mul_ps(qq30
,_mm256_sub_ps(_mm256_mul_ps(rinv30
,rinvsq30
),krf2
));
1230 cutoff_mask
= _mm256_cmp_ps(rsq30
,rcutoff2
,_CMP_LT_OQ
);
1234 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1236 /* Calculate temporary vectorial force */
1237 tx
= _mm256_mul_ps(fscal
,dx30
);
1238 ty
= _mm256_mul_ps(fscal
,dy30
);
1239 tz
= _mm256_mul_ps(fscal
,dz30
);
1241 /* Update vectorial force */
1242 fix3
= _mm256_add_ps(fix3
,tx
);
1243 fiy3
= _mm256_add_ps(fiy3
,ty
);
1244 fiz3
= _mm256_add_ps(fiz3
,tz
);
1246 fjx0
= _mm256_add_ps(fjx0
,tx
);
1247 fjy0
= _mm256_add_ps(fjy0
,ty
);
1248 fjz0
= _mm256_add_ps(fjz0
,tz
);
1252 fjptrA
= f
+j_coord_offsetA
;
1253 fjptrB
= f
+j_coord_offsetB
;
1254 fjptrC
= f
+j_coord_offsetC
;
1255 fjptrD
= f
+j_coord_offsetD
;
1256 fjptrE
= f
+j_coord_offsetE
;
1257 fjptrF
= f
+j_coord_offsetF
;
1258 fjptrG
= f
+j_coord_offsetG
;
1259 fjptrH
= f
+j_coord_offsetH
;
1261 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
1263 /* Inner loop uses 141 flops */
1266 if(jidx
<j_index_end
)
1269 /* Get j neighbor index, and coordinate index */
1270 jnrlistA
= jjnr
[jidx
];
1271 jnrlistB
= jjnr
[jidx
+1];
1272 jnrlistC
= jjnr
[jidx
+2];
1273 jnrlistD
= jjnr
[jidx
+3];
1274 jnrlistE
= jjnr
[jidx
+4];
1275 jnrlistF
= jjnr
[jidx
+5];
1276 jnrlistG
= jjnr
[jidx
+6];
1277 jnrlistH
= jjnr
[jidx
+7];
1278 /* Sign of each element will be negative for non-real atoms.
1279 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1280 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1282 dummy_mask
= gmx_mm256_set_m128(gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
+4)),_mm_setzero_si128())),
1283 gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
)),_mm_setzero_si128())));
1285 jnrA
= (jnrlistA
>=0) ? jnrlistA
: 0;
1286 jnrB
= (jnrlistB
>=0) ? jnrlistB
: 0;
1287 jnrC
= (jnrlistC
>=0) ? jnrlistC
: 0;
1288 jnrD
= (jnrlistD
>=0) ? jnrlistD
: 0;
1289 jnrE
= (jnrlistE
>=0) ? jnrlistE
: 0;
1290 jnrF
= (jnrlistF
>=0) ? jnrlistF
: 0;
1291 jnrG
= (jnrlistG
>=0) ? jnrlistG
: 0;
1292 jnrH
= (jnrlistH
>=0) ? jnrlistH
: 0;
1293 j_coord_offsetA
= DIM
*jnrA
;
1294 j_coord_offsetB
= DIM
*jnrB
;
1295 j_coord_offsetC
= DIM
*jnrC
;
1296 j_coord_offsetD
= DIM
*jnrD
;
1297 j_coord_offsetE
= DIM
*jnrE
;
1298 j_coord_offsetF
= DIM
*jnrF
;
1299 j_coord_offsetG
= DIM
*jnrG
;
1300 j_coord_offsetH
= DIM
*jnrH
;
1302 /* load j atom coordinates */
1303 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
1304 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
1305 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
1306 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
1309 /* Calculate displacement vector */
1310 dx00
= _mm256_sub_ps(ix0
,jx0
);
1311 dy00
= _mm256_sub_ps(iy0
,jy0
);
1312 dz00
= _mm256_sub_ps(iz0
,jz0
);
1313 dx10
= _mm256_sub_ps(ix1
,jx0
);
1314 dy10
= _mm256_sub_ps(iy1
,jy0
);
1315 dz10
= _mm256_sub_ps(iz1
,jz0
);
1316 dx20
= _mm256_sub_ps(ix2
,jx0
);
1317 dy20
= _mm256_sub_ps(iy2
,jy0
);
1318 dz20
= _mm256_sub_ps(iz2
,jz0
);
1319 dx30
= _mm256_sub_ps(ix3
,jx0
);
1320 dy30
= _mm256_sub_ps(iy3
,jy0
);
1321 dz30
= _mm256_sub_ps(iz3
,jz0
);
1323 /* Calculate squared distance and things based on it */
1324 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
1325 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
1326 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
1327 rsq30
= gmx_mm256_calc_rsq_ps(dx30
,dy30
,dz30
);
1329 rinv00
= avx256_invsqrt_f(rsq00
);
1330 rinv10
= avx256_invsqrt_f(rsq10
);
1331 rinv20
= avx256_invsqrt_f(rsq20
);
1332 rinv30
= avx256_invsqrt_f(rsq30
);
1334 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
1335 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
1336 rinvsq30
= _mm256_mul_ps(rinv30
,rinv30
);
1338 /* Load parameters for j particles */
1339 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
1340 charge
+jnrC
+0,charge
+jnrD
+0,
1341 charge
+jnrE
+0,charge
+jnrF
+0,
1342 charge
+jnrG
+0,charge
+jnrH
+0);
1343 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
1344 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
1345 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
1346 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
1347 vdwjidx0E
= 2*vdwtype
[jnrE
+0];
1348 vdwjidx0F
= 2*vdwtype
[jnrF
+0];
1349 vdwjidx0G
= 2*vdwtype
[jnrG
+0];
1350 vdwjidx0H
= 2*vdwtype
[jnrH
+0];
1352 fjx0
= _mm256_setzero_ps();
1353 fjy0
= _mm256_setzero_ps();
1354 fjz0
= _mm256_setzero_ps();
1356 /**************************
1357 * CALCULATE INTERACTIONS *
1358 **************************/
1360 r00
= _mm256_mul_ps(rsq00
,rinv00
);
1361 r00
= _mm256_andnot_ps(dummy_mask
,r00
);
1363 /* Compute parameters for interactions between i and j atoms */
1364 gmx_mm256_load_8pair_swizzle_ps(vdwioffsetptr0
+vdwjidx0A
,
1365 vdwioffsetptr0
+vdwjidx0B
,
1366 vdwioffsetptr0
+vdwjidx0C
,
1367 vdwioffsetptr0
+vdwjidx0D
,
1368 vdwioffsetptr0
+vdwjidx0E
,
1369 vdwioffsetptr0
+vdwjidx0F
,
1370 vdwioffsetptr0
+vdwjidx0G
,
1371 vdwioffsetptr0
+vdwjidx0H
,
1374 /* Calculate table index by multiplying r with table scale and truncate to integer */
1375 rt
= _mm256_mul_ps(r00
,vftabscale
);
1376 vfitab
= _mm256_cvttps_epi32(rt
);
1377 vfeps
= _mm256_sub_ps(rt
,_mm256_round_ps(rt
, _MM_FROUND_FLOOR
));
1378 /* AVX1 does not support 256-bit integer operations, so now we go to 128-bit mode... */
1379 vfitab_lo
= _mm256_extractf128_si256(vfitab
,0x0);
1380 vfitab_hi
= _mm256_extractf128_si256(vfitab
,0x1);
1381 vfitab_lo
= _mm_slli_epi32(vfitab_lo
,3);
1382 vfitab_hi
= _mm_slli_epi32(vfitab_hi
,3);
1384 /* CUBIC SPLINE TABLE DISPERSION */
1385 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
1386 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
1387 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
1388 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
1389 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
1390 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
1391 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
1392 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
1393 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
1394 Heps
= _mm256_mul_ps(vfeps
,H
);
1395 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
1396 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
1397 fvdw6
= _mm256_mul_ps(c6_00
,FF
);
1399 /* CUBIC SPLINE TABLE REPULSION */
1400 vfitab_lo
= _mm_add_epi32(vfitab_lo
,ifour
);
1401 vfitab_hi
= _mm_add_epi32(vfitab_hi
,ifour
);
1402 Y
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,0)),
1403 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,0)));
1404 F
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,1)),
1405 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,1)));
1406 G
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,2)),
1407 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,2)));
1408 H
= gmx_mm256_set_m128(_mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_hi
,3)),
1409 _mm_load_ps(vftab
+ _mm_extract_epi32(vfitab_lo
,3)));
1410 GMX_MM256_HALFTRANSPOSE4_PS(Y
,F
,G
,H
);
1411 Heps
= _mm256_mul_ps(vfeps
,H
);
1412 Fp
= _mm256_add_ps(F
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,Heps
)));
1413 FF
= _mm256_add_ps(Fp
,_mm256_mul_ps(vfeps
,_mm256_add_ps(G
,_mm256_add_ps(Heps
,Heps
))));
1414 fvdw12
= _mm256_mul_ps(c12_00
,FF
);
1415 fvdw
= _mm256_xor_ps(signbit
,_mm256_mul_ps(_mm256_add_ps(fvdw6
,fvdw12
),_mm256_mul_ps(vftabscale
,rinv00
)));
1419 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1421 /* Calculate temporary vectorial force */
1422 tx
= _mm256_mul_ps(fscal
,dx00
);
1423 ty
= _mm256_mul_ps(fscal
,dy00
);
1424 tz
= _mm256_mul_ps(fscal
,dz00
);
1426 /* Update vectorial force */
1427 fix0
= _mm256_add_ps(fix0
,tx
);
1428 fiy0
= _mm256_add_ps(fiy0
,ty
);
1429 fiz0
= _mm256_add_ps(fiz0
,tz
);
1431 fjx0
= _mm256_add_ps(fjx0
,tx
);
1432 fjy0
= _mm256_add_ps(fjy0
,ty
);
1433 fjz0
= _mm256_add_ps(fjz0
,tz
);
1435 /**************************
1436 * CALCULATE INTERACTIONS *
1437 **************************/
1439 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
1442 /* Compute parameters for interactions between i and j atoms */
1443 qq10
= _mm256_mul_ps(iq1
,jq0
);
1445 /* REACTION-FIELD ELECTROSTATICS */
1446 felec
= _mm256_mul_ps(qq10
,_mm256_sub_ps(_mm256_mul_ps(rinv10
,rinvsq10
),krf2
));
1448 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
1452 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1454 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1456 /* Calculate temporary vectorial force */
1457 tx
= _mm256_mul_ps(fscal
,dx10
);
1458 ty
= _mm256_mul_ps(fscal
,dy10
);
1459 tz
= _mm256_mul_ps(fscal
,dz10
);
1461 /* Update vectorial force */
1462 fix1
= _mm256_add_ps(fix1
,tx
);
1463 fiy1
= _mm256_add_ps(fiy1
,ty
);
1464 fiz1
= _mm256_add_ps(fiz1
,tz
);
1466 fjx0
= _mm256_add_ps(fjx0
,tx
);
1467 fjy0
= _mm256_add_ps(fjy0
,ty
);
1468 fjz0
= _mm256_add_ps(fjz0
,tz
);
1472 /**************************
1473 * CALCULATE INTERACTIONS *
1474 **************************/
1476 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
1479 /* Compute parameters for interactions between i and j atoms */
1480 qq20
= _mm256_mul_ps(iq2
,jq0
);
1482 /* REACTION-FIELD ELECTROSTATICS */
1483 felec
= _mm256_mul_ps(qq20
,_mm256_sub_ps(_mm256_mul_ps(rinv20
,rinvsq20
),krf2
));
1485 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
1489 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1491 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1493 /* Calculate temporary vectorial force */
1494 tx
= _mm256_mul_ps(fscal
,dx20
);
1495 ty
= _mm256_mul_ps(fscal
,dy20
);
1496 tz
= _mm256_mul_ps(fscal
,dz20
);
1498 /* Update vectorial force */
1499 fix2
= _mm256_add_ps(fix2
,tx
);
1500 fiy2
= _mm256_add_ps(fiy2
,ty
);
1501 fiz2
= _mm256_add_ps(fiz2
,tz
);
1503 fjx0
= _mm256_add_ps(fjx0
,tx
);
1504 fjy0
= _mm256_add_ps(fjy0
,ty
);
1505 fjz0
= _mm256_add_ps(fjz0
,tz
);
1509 /**************************
1510 * CALCULATE INTERACTIONS *
1511 **************************/
1513 if (gmx_mm256_any_lt(rsq30
,rcutoff2
))
1516 /* Compute parameters for interactions between i and j atoms */
1517 qq30
= _mm256_mul_ps(iq3
,jq0
);
1519 /* REACTION-FIELD ELECTROSTATICS */
1520 felec
= _mm256_mul_ps(qq30
,_mm256_sub_ps(_mm256_mul_ps(rinv30
,rinvsq30
),krf2
));
1522 cutoff_mask
= _mm256_cmp_ps(rsq30
,rcutoff2
,_CMP_LT_OQ
);
1526 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1528 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1530 /* Calculate temporary vectorial force */
1531 tx
= _mm256_mul_ps(fscal
,dx30
);
1532 ty
= _mm256_mul_ps(fscal
,dy30
);
1533 tz
= _mm256_mul_ps(fscal
,dz30
);
1535 /* Update vectorial force */
1536 fix3
= _mm256_add_ps(fix3
,tx
);
1537 fiy3
= _mm256_add_ps(fiy3
,ty
);
1538 fiz3
= _mm256_add_ps(fiz3
,tz
);
1540 fjx0
= _mm256_add_ps(fjx0
,tx
);
1541 fjy0
= _mm256_add_ps(fjy0
,ty
);
1542 fjz0
= _mm256_add_ps(fjz0
,tz
);
1546 fjptrA
= (jnrlistA
>=0) ? f
+j_coord_offsetA
: scratch
;
1547 fjptrB
= (jnrlistB
>=0) ? f
+j_coord_offsetB
: scratch
;
1548 fjptrC
= (jnrlistC
>=0) ? f
+j_coord_offsetC
: scratch
;
1549 fjptrD
= (jnrlistD
>=0) ? f
+j_coord_offsetD
: scratch
;
1550 fjptrE
= (jnrlistE
>=0) ? f
+j_coord_offsetE
: scratch
;
1551 fjptrF
= (jnrlistF
>=0) ? f
+j_coord_offsetF
: scratch
;
1552 fjptrG
= (jnrlistG
>=0) ? f
+j_coord_offsetG
: scratch
;
1553 fjptrH
= (jnrlistH
>=0) ? f
+j_coord_offsetH
: scratch
;
1555 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
1557 /* Inner loop uses 142 flops */
1560 /* End of innermost loop */
1562 gmx_mm256_update_iforce_4atom_swizzle_ps(fix0
,fiy0
,fiz0
,fix1
,fiy1
,fiz1
,fix2
,fiy2
,fiz2
,fix3
,fiy3
,fiz3
,
1563 f
+i_coord_offset
,fshift
+i_shift_offset
);
1565 /* Increment number of inner iterations */
1566 inneriter
+= j_index_end
- j_index_start
;
1568 /* Outer loop uses 24 flops */
1571 /* Increment number of outer iterations */
1574 /* Update outer/inner flops */
1576 inc_nrnb(nrnb
,eNR_NBKERNEL_ELEC_VDW_W4_F
,outeriter
*24 + inneriter
*142);