2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014,2015, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
44 #include "../nb_kernel.h"
45 #include "gromacs/math/vec.h"
46 #include "gromacs/legacyheaders/nrnb.h"
48 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
49 #include "kernelutil_x86_avx_128_fma_single.h"
52 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_VF_avx_128_fma_single
53 * Electrostatics interaction: ReactionField
54 * VdW interaction: CubicSplineTable
55 * Geometry: Water4-Particle
56 * Calculate force/pot: PotentialAndForce
59 nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_VF_avx_128_fma_single
60 (t_nblist
* gmx_restrict nlist
,
61 rvec
* gmx_restrict xx
,
62 rvec
* gmx_restrict ff
,
63 t_forcerec
* gmx_restrict fr
,
64 t_mdatoms
* gmx_restrict mdatoms
,
65 nb_kernel_data_t gmx_unused
* gmx_restrict kernel_data
,
66 t_nrnb
* gmx_restrict nrnb
)
68 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
69 * just 0 for non-waters.
70 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
71 * jnr indices corresponding to data put in the four positions in the SIMD register.
73 int i_shift_offset
,i_coord_offset
,outeriter
,inneriter
;
74 int j_index_start
,j_index_end
,jidx
,nri
,inr
,ggid
,iidx
;
75 int jnrA
,jnrB
,jnrC
,jnrD
;
76 int jnrlistA
,jnrlistB
,jnrlistC
,jnrlistD
;
77 int j_coord_offsetA
,j_coord_offsetB
,j_coord_offsetC
,j_coord_offsetD
;
78 int *iinr
,*jindex
,*jjnr
,*shiftidx
,*gid
;
80 real
*shiftvec
,*fshift
,*x
,*f
;
81 real
*fjptrA
,*fjptrB
,*fjptrC
,*fjptrD
;
83 __m128 fscal
,rcutoff
,rcutoff2
,jidxall
;
85 __m128 ix0
,iy0
,iz0
,fix0
,fiy0
,fiz0
,iq0
,isai0
;
87 __m128 ix1
,iy1
,iz1
,fix1
,fiy1
,fiz1
,iq1
,isai1
;
89 __m128 ix2
,iy2
,iz2
,fix2
,fiy2
,fiz2
,iq2
,isai2
;
91 __m128 ix3
,iy3
,iz3
,fix3
,fiy3
,fiz3
,iq3
,isai3
;
92 int vdwjidx0A
,vdwjidx0B
,vdwjidx0C
,vdwjidx0D
;
93 __m128 jx0
,jy0
,jz0
,fjx0
,fjy0
,fjz0
,jq0
,isaj0
;
94 __m128 dx00
,dy00
,dz00
,rsq00
,rinv00
,rinvsq00
,r00
,qq00
,c6_00
,c12_00
;
95 __m128 dx10
,dy10
,dz10
,rsq10
,rinv10
,rinvsq10
,r10
,qq10
,c6_10
,c12_10
;
96 __m128 dx20
,dy20
,dz20
,rsq20
,rinv20
,rinvsq20
,r20
,qq20
,c6_20
,c12_20
;
97 __m128 dx30
,dy30
,dz30
,rsq30
,rinv30
,rinvsq30
,r30
,qq30
,c6_30
,c12_30
;
98 __m128 velec
,felec
,velecsum
,facel
,crf
,krf
,krf2
;
101 __m128 rinvsix
,rvdw
,vvdw
,vvdw6
,vvdw12
,fvdw
,fvdw6
,fvdw12
,vvdwsum
,sh_vdw_invrcut6
;
104 __m128 one_sixth
= _mm_set1_ps(1.0/6.0);
105 __m128 one_twelfth
= _mm_set1_ps(1.0/12.0);
107 __m128i ifour
= _mm_set1_epi32(4);
108 __m128 rt
,vfeps
,twovfeps
,vftabscale
,Y
,F
,G
,H
,Fp
,VV
,FF
;
110 __m128 dummy_mask
,cutoff_mask
;
111 __m128 signbit
= _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
112 __m128 one
= _mm_set1_ps(1.0);
113 __m128 two
= _mm_set1_ps(2.0);
119 jindex
= nlist
->jindex
;
121 shiftidx
= nlist
->shift
;
123 shiftvec
= fr
->shift_vec
[0];
124 fshift
= fr
->fshift
[0];
125 facel
= _mm_set1_ps(fr
->epsfac
);
126 charge
= mdatoms
->chargeA
;
127 krf
= _mm_set1_ps(fr
->ic
->k_rf
);
128 krf2
= _mm_set1_ps(fr
->ic
->k_rf
*2.0);
129 crf
= _mm_set1_ps(fr
->ic
->c_rf
);
130 nvdwtype
= fr
->ntype
;
132 vdwtype
= mdatoms
->typeA
;
134 vftab
= kernel_data
->table_vdw
->data
;
135 vftabscale
= _mm_set1_ps(kernel_data
->table_vdw
->scale
);
137 /* Setup water-specific parameters */
138 inr
= nlist
->iinr
[0];
139 iq1
= _mm_mul_ps(facel
,_mm_set1_ps(charge
[inr
+1]));
140 iq2
= _mm_mul_ps(facel
,_mm_set1_ps(charge
[inr
+2]));
141 iq3
= _mm_mul_ps(facel
,_mm_set1_ps(charge
[inr
+3]));
142 vdwioffset0
= 2*nvdwtype
*vdwtype
[inr
+0];
144 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
145 rcutoff_scalar
= fr
->rcoulomb
;
146 rcutoff
= _mm_set1_ps(rcutoff_scalar
);
147 rcutoff2
= _mm_mul_ps(rcutoff
,rcutoff
);
149 /* Avoid stupid compiler warnings */
150 jnrA
= jnrB
= jnrC
= jnrD
= 0;
159 for(iidx
=0;iidx
<4*DIM
;iidx
++)
164 /* Start outer loop over neighborlists */
165 for(iidx
=0; iidx
<nri
; iidx
++)
167 /* Load shift vector for this list */
168 i_shift_offset
= DIM
*shiftidx
[iidx
];
170 /* Load limits for loop over neighbors */
171 j_index_start
= jindex
[iidx
];
172 j_index_end
= jindex
[iidx
+1];
174 /* Get outer coordinate index */
176 i_coord_offset
= DIM
*inr
;
178 /* Load i particle coords and add shift vector */
179 gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec
+i_shift_offset
,x
+i_coord_offset
,
180 &ix0
,&iy0
,&iz0
,&ix1
,&iy1
,&iz1
,&ix2
,&iy2
,&iz2
,&ix3
,&iy3
,&iz3
);
182 fix0
= _mm_setzero_ps();
183 fiy0
= _mm_setzero_ps();
184 fiz0
= _mm_setzero_ps();
185 fix1
= _mm_setzero_ps();
186 fiy1
= _mm_setzero_ps();
187 fiz1
= _mm_setzero_ps();
188 fix2
= _mm_setzero_ps();
189 fiy2
= _mm_setzero_ps();
190 fiz2
= _mm_setzero_ps();
191 fix3
= _mm_setzero_ps();
192 fiy3
= _mm_setzero_ps();
193 fiz3
= _mm_setzero_ps();
195 /* Reset potential sums */
196 velecsum
= _mm_setzero_ps();
197 vvdwsum
= _mm_setzero_ps();
199 /* Start inner kernel loop */
200 for(jidx
=j_index_start
; jidx
<j_index_end
&& jjnr
[jidx
+3]>=0; jidx
+=4)
203 /* Get j neighbor index, and coordinate index */
208 j_coord_offsetA
= DIM
*jnrA
;
209 j_coord_offsetB
= DIM
*jnrB
;
210 j_coord_offsetC
= DIM
*jnrC
;
211 j_coord_offsetD
= DIM
*jnrD
;
213 /* load j atom coordinates */
214 gmx_mm_load_1rvec_4ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
215 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
218 /* Calculate displacement vector */
219 dx00
= _mm_sub_ps(ix0
,jx0
);
220 dy00
= _mm_sub_ps(iy0
,jy0
);
221 dz00
= _mm_sub_ps(iz0
,jz0
);
222 dx10
= _mm_sub_ps(ix1
,jx0
);
223 dy10
= _mm_sub_ps(iy1
,jy0
);
224 dz10
= _mm_sub_ps(iz1
,jz0
);
225 dx20
= _mm_sub_ps(ix2
,jx0
);
226 dy20
= _mm_sub_ps(iy2
,jy0
);
227 dz20
= _mm_sub_ps(iz2
,jz0
);
228 dx30
= _mm_sub_ps(ix3
,jx0
);
229 dy30
= _mm_sub_ps(iy3
,jy0
);
230 dz30
= _mm_sub_ps(iz3
,jz0
);
232 /* Calculate squared distance and things based on it */
233 rsq00
= gmx_mm_calc_rsq_ps(dx00
,dy00
,dz00
);
234 rsq10
= gmx_mm_calc_rsq_ps(dx10
,dy10
,dz10
);
235 rsq20
= gmx_mm_calc_rsq_ps(dx20
,dy20
,dz20
);
236 rsq30
= gmx_mm_calc_rsq_ps(dx30
,dy30
,dz30
);
238 rinv00
= gmx_mm_invsqrt_ps(rsq00
);
239 rinv10
= gmx_mm_invsqrt_ps(rsq10
);
240 rinv20
= gmx_mm_invsqrt_ps(rsq20
);
241 rinv30
= gmx_mm_invsqrt_ps(rsq30
);
243 rinvsq10
= _mm_mul_ps(rinv10
,rinv10
);
244 rinvsq20
= _mm_mul_ps(rinv20
,rinv20
);
245 rinvsq30
= _mm_mul_ps(rinv30
,rinv30
);
247 /* Load parameters for j particles */
248 jq0
= gmx_mm_load_4real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
249 charge
+jnrC
+0,charge
+jnrD
+0);
250 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
251 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
252 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
253 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
255 fjx0
= _mm_setzero_ps();
256 fjy0
= _mm_setzero_ps();
257 fjz0
= _mm_setzero_ps();
259 /**************************
260 * CALCULATE INTERACTIONS *
261 **************************/
263 r00
= _mm_mul_ps(rsq00
,rinv00
);
265 /* Compute parameters for interactions between i and j atoms */
266 gmx_mm_load_4pair_swizzle_ps(vdwparam
+vdwioffset0
+vdwjidx0A
,
267 vdwparam
+vdwioffset0
+vdwjidx0B
,
268 vdwparam
+vdwioffset0
+vdwjidx0C
,
269 vdwparam
+vdwioffset0
+vdwjidx0D
,
272 /* Calculate table index by multiplying r with table scale and truncate to integer */
273 rt
= _mm_mul_ps(r00
,vftabscale
);
274 vfitab
= _mm_cvttps_epi32(rt
);
276 vfeps
= _mm_frcz_ps(rt
);
278 vfeps
= _mm_sub_ps(rt
,_mm_round_ps(rt
, _MM_FROUND_FLOOR
));
280 twovfeps
= _mm_add_ps(vfeps
,vfeps
);
281 vfitab
= _mm_slli_epi32(vfitab
,3);
283 /* CUBIC SPLINE TABLE DISPERSION */
284 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
285 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
286 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
287 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
288 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
289 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
290 VV
= _mm_macc_ps(vfeps
,Fp
,Y
);
291 vvdw6
= _mm_mul_ps(c6_00
,VV
);
292 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
293 fvdw6
= _mm_mul_ps(c6_00
,FF
);
295 /* CUBIC SPLINE TABLE REPULSION */
296 vfitab
= _mm_add_epi32(vfitab
,ifour
);
297 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
298 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
299 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
300 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
301 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
302 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
303 VV
= _mm_macc_ps(vfeps
,Fp
,Y
);
304 vvdw12
= _mm_mul_ps(c12_00
,VV
);
305 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
306 fvdw12
= _mm_mul_ps(c12_00
,FF
);
307 vvdw
= _mm_add_ps(vvdw12
,vvdw6
);
308 fvdw
= _mm_xor_ps(signbit
,_mm_mul_ps(_mm_add_ps(fvdw6
,fvdw12
),_mm_mul_ps(vftabscale
,rinv00
)));
310 /* Update potential sum for this i atom from the interaction with this j atom. */
311 vvdwsum
= _mm_add_ps(vvdwsum
,vvdw
);
315 /* Update vectorial force */
316 fix0
= _mm_macc_ps(dx00
,fscal
,fix0
);
317 fiy0
= _mm_macc_ps(dy00
,fscal
,fiy0
);
318 fiz0
= _mm_macc_ps(dz00
,fscal
,fiz0
);
320 fjx0
= _mm_macc_ps(dx00
,fscal
,fjx0
);
321 fjy0
= _mm_macc_ps(dy00
,fscal
,fjy0
);
322 fjz0
= _mm_macc_ps(dz00
,fscal
,fjz0
);
324 /**************************
325 * CALCULATE INTERACTIONS *
326 **************************/
328 if (gmx_mm_any_lt(rsq10
,rcutoff2
))
331 /* Compute parameters for interactions between i and j atoms */
332 qq10
= _mm_mul_ps(iq1
,jq0
);
334 /* REACTION-FIELD ELECTROSTATICS */
335 velec
= _mm_mul_ps(qq10
,_mm_sub_ps(_mm_macc_ps(krf
,rsq10
,rinv10
),crf
));
336 felec
= _mm_mul_ps(qq10
,_mm_msub_ps(rinv10
,rinvsq10
,krf2
));
338 cutoff_mask
= _mm_cmplt_ps(rsq10
,rcutoff2
);
340 /* Update potential sum for this i atom from the interaction with this j atom. */
341 velec
= _mm_and_ps(velec
,cutoff_mask
);
342 velecsum
= _mm_add_ps(velecsum
,velec
);
346 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
348 /* Update vectorial force */
349 fix1
= _mm_macc_ps(dx10
,fscal
,fix1
);
350 fiy1
= _mm_macc_ps(dy10
,fscal
,fiy1
);
351 fiz1
= _mm_macc_ps(dz10
,fscal
,fiz1
);
353 fjx0
= _mm_macc_ps(dx10
,fscal
,fjx0
);
354 fjy0
= _mm_macc_ps(dy10
,fscal
,fjy0
);
355 fjz0
= _mm_macc_ps(dz10
,fscal
,fjz0
);
359 /**************************
360 * CALCULATE INTERACTIONS *
361 **************************/
363 if (gmx_mm_any_lt(rsq20
,rcutoff2
))
366 /* Compute parameters for interactions between i and j atoms */
367 qq20
= _mm_mul_ps(iq2
,jq0
);
369 /* REACTION-FIELD ELECTROSTATICS */
370 velec
= _mm_mul_ps(qq20
,_mm_sub_ps(_mm_macc_ps(krf
,rsq20
,rinv20
),crf
));
371 felec
= _mm_mul_ps(qq20
,_mm_msub_ps(rinv20
,rinvsq20
,krf2
));
373 cutoff_mask
= _mm_cmplt_ps(rsq20
,rcutoff2
);
375 /* Update potential sum for this i atom from the interaction with this j atom. */
376 velec
= _mm_and_ps(velec
,cutoff_mask
);
377 velecsum
= _mm_add_ps(velecsum
,velec
);
381 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
383 /* Update vectorial force */
384 fix2
= _mm_macc_ps(dx20
,fscal
,fix2
);
385 fiy2
= _mm_macc_ps(dy20
,fscal
,fiy2
);
386 fiz2
= _mm_macc_ps(dz20
,fscal
,fiz2
);
388 fjx0
= _mm_macc_ps(dx20
,fscal
,fjx0
);
389 fjy0
= _mm_macc_ps(dy20
,fscal
,fjy0
);
390 fjz0
= _mm_macc_ps(dz20
,fscal
,fjz0
);
394 /**************************
395 * CALCULATE INTERACTIONS *
396 **************************/
398 if (gmx_mm_any_lt(rsq30
,rcutoff2
))
401 /* Compute parameters for interactions between i and j atoms */
402 qq30
= _mm_mul_ps(iq3
,jq0
);
404 /* REACTION-FIELD ELECTROSTATICS */
405 velec
= _mm_mul_ps(qq30
,_mm_sub_ps(_mm_macc_ps(krf
,rsq30
,rinv30
),crf
));
406 felec
= _mm_mul_ps(qq30
,_mm_msub_ps(rinv30
,rinvsq30
,krf2
));
408 cutoff_mask
= _mm_cmplt_ps(rsq30
,rcutoff2
);
410 /* Update potential sum for this i atom from the interaction with this j atom. */
411 velec
= _mm_and_ps(velec
,cutoff_mask
);
412 velecsum
= _mm_add_ps(velecsum
,velec
);
416 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
418 /* Update vectorial force */
419 fix3
= _mm_macc_ps(dx30
,fscal
,fix3
);
420 fiy3
= _mm_macc_ps(dy30
,fscal
,fiy3
);
421 fiz3
= _mm_macc_ps(dz30
,fscal
,fiz3
);
423 fjx0
= _mm_macc_ps(dx30
,fscal
,fjx0
);
424 fjy0
= _mm_macc_ps(dy30
,fscal
,fjy0
);
425 fjz0
= _mm_macc_ps(dz30
,fscal
,fjz0
);
429 fjptrA
= f
+j_coord_offsetA
;
430 fjptrB
= f
+j_coord_offsetB
;
431 fjptrC
= f
+j_coord_offsetC
;
432 fjptrD
= f
+j_coord_offsetD
;
434 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjx0
,fjy0
,fjz0
);
436 /* Inner loop uses 176 flops */
442 /* Get j neighbor index, and coordinate index */
443 jnrlistA
= jjnr
[jidx
];
444 jnrlistB
= jjnr
[jidx
+1];
445 jnrlistC
= jjnr
[jidx
+2];
446 jnrlistD
= jjnr
[jidx
+3];
447 /* Sign of each element will be negative for non-real atoms.
448 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
449 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
451 dummy_mask
= gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
)),_mm_setzero_si128()));
452 jnrA
= (jnrlistA
>=0) ? jnrlistA
: 0;
453 jnrB
= (jnrlistB
>=0) ? jnrlistB
: 0;
454 jnrC
= (jnrlistC
>=0) ? jnrlistC
: 0;
455 jnrD
= (jnrlistD
>=0) ? jnrlistD
: 0;
456 j_coord_offsetA
= DIM
*jnrA
;
457 j_coord_offsetB
= DIM
*jnrB
;
458 j_coord_offsetC
= DIM
*jnrC
;
459 j_coord_offsetD
= DIM
*jnrD
;
461 /* load j atom coordinates */
462 gmx_mm_load_1rvec_4ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
463 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
466 /* Calculate displacement vector */
467 dx00
= _mm_sub_ps(ix0
,jx0
);
468 dy00
= _mm_sub_ps(iy0
,jy0
);
469 dz00
= _mm_sub_ps(iz0
,jz0
);
470 dx10
= _mm_sub_ps(ix1
,jx0
);
471 dy10
= _mm_sub_ps(iy1
,jy0
);
472 dz10
= _mm_sub_ps(iz1
,jz0
);
473 dx20
= _mm_sub_ps(ix2
,jx0
);
474 dy20
= _mm_sub_ps(iy2
,jy0
);
475 dz20
= _mm_sub_ps(iz2
,jz0
);
476 dx30
= _mm_sub_ps(ix3
,jx0
);
477 dy30
= _mm_sub_ps(iy3
,jy0
);
478 dz30
= _mm_sub_ps(iz3
,jz0
);
480 /* Calculate squared distance and things based on it */
481 rsq00
= gmx_mm_calc_rsq_ps(dx00
,dy00
,dz00
);
482 rsq10
= gmx_mm_calc_rsq_ps(dx10
,dy10
,dz10
);
483 rsq20
= gmx_mm_calc_rsq_ps(dx20
,dy20
,dz20
);
484 rsq30
= gmx_mm_calc_rsq_ps(dx30
,dy30
,dz30
);
486 rinv00
= gmx_mm_invsqrt_ps(rsq00
);
487 rinv10
= gmx_mm_invsqrt_ps(rsq10
);
488 rinv20
= gmx_mm_invsqrt_ps(rsq20
);
489 rinv30
= gmx_mm_invsqrt_ps(rsq30
);
491 rinvsq10
= _mm_mul_ps(rinv10
,rinv10
);
492 rinvsq20
= _mm_mul_ps(rinv20
,rinv20
);
493 rinvsq30
= _mm_mul_ps(rinv30
,rinv30
);
495 /* Load parameters for j particles */
496 jq0
= gmx_mm_load_4real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
497 charge
+jnrC
+0,charge
+jnrD
+0);
498 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
499 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
500 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
501 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
503 fjx0
= _mm_setzero_ps();
504 fjy0
= _mm_setzero_ps();
505 fjz0
= _mm_setzero_ps();
507 /**************************
508 * CALCULATE INTERACTIONS *
509 **************************/
511 r00
= _mm_mul_ps(rsq00
,rinv00
);
512 r00
= _mm_andnot_ps(dummy_mask
,r00
);
514 /* Compute parameters for interactions between i and j atoms */
515 gmx_mm_load_4pair_swizzle_ps(vdwparam
+vdwioffset0
+vdwjidx0A
,
516 vdwparam
+vdwioffset0
+vdwjidx0B
,
517 vdwparam
+vdwioffset0
+vdwjidx0C
,
518 vdwparam
+vdwioffset0
+vdwjidx0D
,
521 /* Calculate table index by multiplying r with table scale and truncate to integer */
522 rt
= _mm_mul_ps(r00
,vftabscale
);
523 vfitab
= _mm_cvttps_epi32(rt
);
525 vfeps
= _mm_frcz_ps(rt
);
527 vfeps
= _mm_sub_ps(rt
,_mm_round_ps(rt
, _MM_FROUND_FLOOR
));
529 twovfeps
= _mm_add_ps(vfeps
,vfeps
);
530 vfitab
= _mm_slli_epi32(vfitab
,3);
532 /* CUBIC SPLINE TABLE DISPERSION */
533 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
534 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
535 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
536 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
537 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
538 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
539 VV
= _mm_macc_ps(vfeps
,Fp
,Y
);
540 vvdw6
= _mm_mul_ps(c6_00
,VV
);
541 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
542 fvdw6
= _mm_mul_ps(c6_00
,FF
);
544 /* CUBIC SPLINE TABLE REPULSION */
545 vfitab
= _mm_add_epi32(vfitab
,ifour
);
546 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
547 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
548 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
549 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
550 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
551 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
552 VV
= _mm_macc_ps(vfeps
,Fp
,Y
);
553 vvdw12
= _mm_mul_ps(c12_00
,VV
);
554 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
555 fvdw12
= _mm_mul_ps(c12_00
,FF
);
556 vvdw
= _mm_add_ps(vvdw12
,vvdw6
);
557 fvdw
= _mm_xor_ps(signbit
,_mm_mul_ps(_mm_add_ps(fvdw6
,fvdw12
),_mm_mul_ps(vftabscale
,rinv00
)));
559 /* Update potential sum for this i atom from the interaction with this j atom. */
560 vvdw
= _mm_andnot_ps(dummy_mask
,vvdw
);
561 vvdwsum
= _mm_add_ps(vvdwsum
,vvdw
);
565 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
567 /* Update vectorial force */
568 fix0
= _mm_macc_ps(dx00
,fscal
,fix0
);
569 fiy0
= _mm_macc_ps(dy00
,fscal
,fiy0
);
570 fiz0
= _mm_macc_ps(dz00
,fscal
,fiz0
);
572 fjx0
= _mm_macc_ps(dx00
,fscal
,fjx0
);
573 fjy0
= _mm_macc_ps(dy00
,fscal
,fjy0
);
574 fjz0
= _mm_macc_ps(dz00
,fscal
,fjz0
);
576 /**************************
577 * CALCULATE INTERACTIONS *
578 **************************/
580 if (gmx_mm_any_lt(rsq10
,rcutoff2
))
583 /* Compute parameters for interactions between i and j atoms */
584 qq10
= _mm_mul_ps(iq1
,jq0
);
586 /* REACTION-FIELD ELECTROSTATICS */
587 velec
= _mm_mul_ps(qq10
,_mm_sub_ps(_mm_macc_ps(krf
,rsq10
,rinv10
),crf
));
588 felec
= _mm_mul_ps(qq10
,_mm_msub_ps(rinv10
,rinvsq10
,krf2
));
590 cutoff_mask
= _mm_cmplt_ps(rsq10
,rcutoff2
);
592 /* Update potential sum for this i atom from the interaction with this j atom. */
593 velec
= _mm_and_ps(velec
,cutoff_mask
);
594 velec
= _mm_andnot_ps(dummy_mask
,velec
);
595 velecsum
= _mm_add_ps(velecsum
,velec
);
599 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
601 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
603 /* Update vectorial force */
604 fix1
= _mm_macc_ps(dx10
,fscal
,fix1
);
605 fiy1
= _mm_macc_ps(dy10
,fscal
,fiy1
);
606 fiz1
= _mm_macc_ps(dz10
,fscal
,fiz1
);
608 fjx0
= _mm_macc_ps(dx10
,fscal
,fjx0
);
609 fjy0
= _mm_macc_ps(dy10
,fscal
,fjy0
);
610 fjz0
= _mm_macc_ps(dz10
,fscal
,fjz0
);
614 /**************************
615 * CALCULATE INTERACTIONS *
616 **************************/
618 if (gmx_mm_any_lt(rsq20
,rcutoff2
))
621 /* Compute parameters for interactions between i and j atoms */
622 qq20
= _mm_mul_ps(iq2
,jq0
);
624 /* REACTION-FIELD ELECTROSTATICS */
625 velec
= _mm_mul_ps(qq20
,_mm_sub_ps(_mm_macc_ps(krf
,rsq20
,rinv20
),crf
));
626 felec
= _mm_mul_ps(qq20
,_mm_msub_ps(rinv20
,rinvsq20
,krf2
));
628 cutoff_mask
= _mm_cmplt_ps(rsq20
,rcutoff2
);
630 /* Update potential sum for this i atom from the interaction with this j atom. */
631 velec
= _mm_and_ps(velec
,cutoff_mask
);
632 velec
= _mm_andnot_ps(dummy_mask
,velec
);
633 velecsum
= _mm_add_ps(velecsum
,velec
);
637 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
639 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
641 /* Update vectorial force */
642 fix2
= _mm_macc_ps(dx20
,fscal
,fix2
);
643 fiy2
= _mm_macc_ps(dy20
,fscal
,fiy2
);
644 fiz2
= _mm_macc_ps(dz20
,fscal
,fiz2
);
646 fjx0
= _mm_macc_ps(dx20
,fscal
,fjx0
);
647 fjy0
= _mm_macc_ps(dy20
,fscal
,fjy0
);
648 fjz0
= _mm_macc_ps(dz20
,fscal
,fjz0
);
652 /**************************
653 * CALCULATE INTERACTIONS *
654 **************************/
656 if (gmx_mm_any_lt(rsq30
,rcutoff2
))
659 /* Compute parameters for interactions between i and j atoms */
660 qq30
= _mm_mul_ps(iq3
,jq0
);
662 /* REACTION-FIELD ELECTROSTATICS */
663 velec
= _mm_mul_ps(qq30
,_mm_sub_ps(_mm_macc_ps(krf
,rsq30
,rinv30
),crf
));
664 felec
= _mm_mul_ps(qq30
,_mm_msub_ps(rinv30
,rinvsq30
,krf2
));
666 cutoff_mask
= _mm_cmplt_ps(rsq30
,rcutoff2
);
668 /* Update potential sum for this i atom from the interaction with this j atom. */
669 velec
= _mm_and_ps(velec
,cutoff_mask
);
670 velec
= _mm_andnot_ps(dummy_mask
,velec
);
671 velecsum
= _mm_add_ps(velecsum
,velec
);
675 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
677 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
679 /* Update vectorial force */
680 fix3
= _mm_macc_ps(dx30
,fscal
,fix3
);
681 fiy3
= _mm_macc_ps(dy30
,fscal
,fiy3
);
682 fiz3
= _mm_macc_ps(dz30
,fscal
,fiz3
);
684 fjx0
= _mm_macc_ps(dx30
,fscal
,fjx0
);
685 fjy0
= _mm_macc_ps(dy30
,fscal
,fjy0
);
686 fjz0
= _mm_macc_ps(dz30
,fscal
,fjz0
);
690 fjptrA
= (jnrlistA
>=0) ? f
+j_coord_offsetA
: scratch
;
691 fjptrB
= (jnrlistB
>=0) ? f
+j_coord_offsetB
: scratch
;
692 fjptrC
= (jnrlistC
>=0) ? f
+j_coord_offsetC
: scratch
;
693 fjptrD
= (jnrlistD
>=0) ? f
+j_coord_offsetD
: scratch
;
695 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjx0
,fjy0
,fjz0
);
697 /* Inner loop uses 177 flops */
700 /* End of innermost loop */
702 gmx_mm_update_iforce_4atom_swizzle_ps(fix0
,fiy0
,fiz0
,fix1
,fiy1
,fiz1
,fix2
,fiy2
,fiz2
,fix3
,fiy3
,fiz3
,
703 f
+i_coord_offset
,fshift
+i_shift_offset
);
706 /* Update potential energies */
707 gmx_mm_update_1pot_ps(velecsum
,kernel_data
->energygrp_elec
+ggid
);
708 gmx_mm_update_1pot_ps(vvdwsum
,kernel_data
->energygrp_vdw
+ggid
);
710 /* Increment number of inner iterations */
711 inneriter
+= j_index_end
- j_index_start
;
713 /* Outer loop uses 26 flops */
716 /* Increment number of outer iterations */
719 /* Update outer/inner flops */
721 inc_nrnb(nrnb
,eNR_NBKERNEL_ELEC_VDW_W4_VF
,outeriter
*26 + inneriter
*177);
724 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_F_avx_128_fma_single
725 * Electrostatics interaction: ReactionField
726 * VdW interaction: CubicSplineTable
727 * Geometry: Water4-Particle
728 * Calculate force/pot: Force
731 nb_kernel_ElecRFCut_VdwCSTab_GeomW4P1_F_avx_128_fma_single
732 (t_nblist
* gmx_restrict nlist
,
733 rvec
* gmx_restrict xx
,
734 rvec
* gmx_restrict ff
,
735 t_forcerec
* gmx_restrict fr
,
736 t_mdatoms
* gmx_restrict mdatoms
,
737 nb_kernel_data_t gmx_unused
* gmx_restrict kernel_data
,
738 t_nrnb
* gmx_restrict nrnb
)
740 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
741 * just 0 for non-waters.
742 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
743 * jnr indices corresponding to data put in the four positions in the SIMD register.
745 int i_shift_offset
,i_coord_offset
,outeriter
,inneriter
;
746 int j_index_start
,j_index_end
,jidx
,nri
,inr
,ggid
,iidx
;
747 int jnrA
,jnrB
,jnrC
,jnrD
;
748 int jnrlistA
,jnrlistB
,jnrlistC
,jnrlistD
;
749 int j_coord_offsetA
,j_coord_offsetB
,j_coord_offsetC
,j_coord_offsetD
;
750 int *iinr
,*jindex
,*jjnr
,*shiftidx
,*gid
;
752 real
*shiftvec
,*fshift
,*x
,*f
;
753 real
*fjptrA
,*fjptrB
,*fjptrC
,*fjptrD
;
755 __m128 fscal
,rcutoff
,rcutoff2
,jidxall
;
757 __m128 ix0
,iy0
,iz0
,fix0
,fiy0
,fiz0
,iq0
,isai0
;
759 __m128 ix1
,iy1
,iz1
,fix1
,fiy1
,fiz1
,iq1
,isai1
;
761 __m128 ix2
,iy2
,iz2
,fix2
,fiy2
,fiz2
,iq2
,isai2
;
763 __m128 ix3
,iy3
,iz3
,fix3
,fiy3
,fiz3
,iq3
,isai3
;
764 int vdwjidx0A
,vdwjidx0B
,vdwjidx0C
,vdwjidx0D
;
765 __m128 jx0
,jy0
,jz0
,fjx0
,fjy0
,fjz0
,jq0
,isaj0
;
766 __m128 dx00
,dy00
,dz00
,rsq00
,rinv00
,rinvsq00
,r00
,qq00
,c6_00
,c12_00
;
767 __m128 dx10
,dy10
,dz10
,rsq10
,rinv10
,rinvsq10
,r10
,qq10
,c6_10
,c12_10
;
768 __m128 dx20
,dy20
,dz20
,rsq20
,rinv20
,rinvsq20
,r20
,qq20
,c6_20
,c12_20
;
769 __m128 dx30
,dy30
,dz30
,rsq30
,rinv30
,rinvsq30
,r30
,qq30
,c6_30
,c12_30
;
770 __m128 velec
,felec
,velecsum
,facel
,crf
,krf
,krf2
;
773 __m128 rinvsix
,rvdw
,vvdw
,vvdw6
,vvdw12
,fvdw
,fvdw6
,fvdw12
,vvdwsum
,sh_vdw_invrcut6
;
776 __m128 one_sixth
= _mm_set1_ps(1.0/6.0);
777 __m128 one_twelfth
= _mm_set1_ps(1.0/12.0);
779 __m128i ifour
= _mm_set1_epi32(4);
780 __m128 rt
,vfeps
,twovfeps
,vftabscale
,Y
,F
,G
,H
,Fp
,VV
,FF
;
782 __m128 dummy_mask
,cutoff_mask
;
783 __m128 signbit
= _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
784 __m128 one
= _mm_set1_ps(1.0);
785 __m128 two
= _mm_set1_ps(2.0);
791 jindex
= nlist
->jindex
;
793 shiftidx
= nlist
->shift
;
795 shiftvec
= fr
->shift_vec
[0];
796 fshift
= fr
->fshift
[0];
797 facel
= _mm_set1_ps(fr
->epsfac
);
798 charge
= mdatoms
->chargeA
;
799 krf
= _mm_set1_ps(fr
->ic
->k_rf
);
800 krf2
= _mm_set1_ps(fr
->ic
->k_rf
*2.0);
801 crf
= _mm_set1_ps(fr
->ic
->c_rf
);
802 nvdwtype
= fr
->ntype
;
804 vdwtype
= mdatoms
->typeA
;
806 vftab
= kernel_data
->table_vdw
->data
;
807 vftabscale
= _mm_set1_ps(kernel_data
->table_vdw
->scale
);
809 /* Setup water-specific parameters */
810 inr
= nlist
->iinr
[0];
811 iq1
= _mm_mul_ps(facel
,_mm_set1_ps(charge
[inr
+1]));
812 iq2
= _mm_mul_ps(facel
,_mm_set1_ps(charge
[inr
+2]));
813 iq3
= _mm_mul_ps(facel
,_mm_set1_ps(charge
[inr
+3]));
814 vdwioffset0
= 2*nvdwtype
*vdwtype
[inr
+0];
816 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
817 rcutoff_scalar
= fr
->rcoulomb
;
818 rcutoff
= _mm_set1_ps(rcutoff_scalar
);
819 rcutoff2
= _mm_mul_ps(rcutoff
,rcutoff
);
821 /* Avoid stupid compiler warnings */
822 jnrA
= jnrB
= jnrC
= jnrD
= 0;
831 for(iidx
=0;iidx
<4*DIM
;iidx
++)
836 /* Start outer loop over neighborlists */
837 for(iidx
=0; iidx
<nri
; iidx
++)
839 /* Load shift vector for this list */
840 i_shift_offset
= DIM
*shiftidx
[iidx
];
842 /* Load limits for loop over neighbors */
843 j_index_start
= jindex
[iidx
];
844 j_index_end
= jindex
[iidx
+1];
846 /* Get outer coordinate index */
848 i_coord_offset
= DIM
*inr
;
850 /* Load i particle coords and add shift vector */
851 gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec
+i_shift_offset
,x
+i_coord_offset
,
852 &ix0
,&iy0
,&iz0
,&ix1
,&iy1
,&iz1
,&ix2
,&iy2
,&iz2
,&ix3
,&iy3
,&iz3
);
854 fix0
= _mm_setzero_ps();
855 fiy0
= _mm_setzero_ps();
856 fiz0
= _mm_setzero_ps();
857 fix1
= _mm_setzero_ps();
858 fiy1
= _mm_setzero_ps();
859 fiz1
= _mm_setzero_ps();
860 fix2
= _mm_setzero_ps();
861 fiy2
= _mm_setzero_ps();
862 fiz2
= _mm_setzero_ps();
863 fix3
= _mm_setzero_ps();
864 fiy3
= _mm_setzero_ps();
865 fiz3
= _mm_setzero_ps();
867 /* Start inner kernel loop */
868 for(jidx
=j_index_start
; jidx
<j_index_end
&& jjnr
[jidx
+3]>=0; jidx
+=4)
871 /* Get j neighbor index, and coordinate index */
876 j_coord_offsetA
= DIM
*jnrA
;
877 j_coord_offsetB
= DIM
*jnrB
;
878 j_coord_offsetC
= DIM
*jnrC
;
879 j_coord_offsetD
= DIM
*jnrD
;
881 /* load j atom coordinates */
882 gmx_mm_load_1rvec_4ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
883 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
886 /* Calculate displacement vector */
887 dx00
= _mm_sub_ps(ix0
,jx0
);
888 dy00
= _mm_sub_ps(iy0
,jy0
);
889 dz00
= _mm_sub_ps(iz0
,jz0
);
890 dx10
= _mm_sub_ps(ix1
,jx0
);
891 dy10
= _mm_sub_ps(iy1
,jy0
);
892 dz10
= _mm_sub_ps(iz1
,jz0
);
893 dx20
= _mm_sub_ps(ix2
,jx0
);
894 dy20
= _mm_sub_ps(iy2
,jy0
);
895 dz20
= _mm_sub_ps(iz2
,jz0
);
896 dx30
= _mm_sub_ps(ix3
,jx0
);
897 dy30
= _mm_sub_ps(iy3
,jy0
);
898 dz30
= _mm_sub_ps(iz3
,jz0
);
900 /* Calculate squared distance and things based on it */
901 rsq00
= gmx_mm_calc_rsq_ps(dx00
,dy00
,dz00
);
902 rsq10
= gmx_mm_calc_rsq_ps(dx10
,dy10
,dz10
);
903 rsq20
= gmx_mm_calc_rsq_ps(dx20
,dy20
,dz20
);
904 rsq30
= gmx_mm_calc_rsq_ps(dx30
,dy30
,dz30
);
906 rinv00
= gmx_mm_invsqrt_ps(rsq00
);
907 rinv10
= gmx_mm_invsqrt_ps(rsq10
);
908 rinv20
= gmx_mm_invsqrt_ps(rsq20
);
909 rinv30
= gmx_mm_invsqrt_ps(rsq30
);
911 rinvsq10
= _mm_mul_ps(rinv10
,rinv10
);
912 rinvsq20
= _mm_mul_ps(rinv20
,rinv20
);
913 rinvsq30
= _mm_mul_ps(rinv30
,rinv30
);
915 /* Load parameters for j particles */
916 jq0
= gmx_mm_load_4real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
917 charge
+jnrC
+0,charge
+jnrD
+0);
918 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
919 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
920 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
921 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
923 fjx0
= _mm_setzero_ps();
924 fjy0
= _mm_setzero_ps();
925 fjz0
= _mm_setzero_ps();
927 /**************************
928 * CALCULATE INTERACTIONS *
929 **************************/
931 r00
= _mm_mul_ps(rsq00
,rinv00
);
933 /* Compute parameters for interactions between i and j atoms */
934 gmx_mm_load_4pair_swizzle_ps(vdwparam
+vdwioffset0
+vdwjidx0A
,
935 vdwparam
+vdwioffset0
+vdwjidx0B
,
936 vdwparam
+vdwioffset0
+vdwjidx0C
,
937 vdwparam
+vdwioffset0
+vdwjidx0D
,
940 /* Calculate table index by multiplying r with table scale and truncate to integer */
941 rt
= _mm_mul_ps(r00
,vftabscale
);
942 vfitab
= _mm_cvttps_epi32(rt
);
944 vfeps
= _mm_frcz_ps(rt
);
946 vfeps
= _mm_sub_ps(rt
,_mm_round_ps(rt
, _MM_FROUND_FLOOR
));
948 twovfeps
= _mm_add_ps(vfeps
,vfeps
);
949 vfitab
= _mm_slli_epi32(vfitab
,3);
951 /* CUBIC SPLINE TABLE DISPERSION */
952 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
953 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
954 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
955 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
956 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
957 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
958 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
959 fvdw6
= _mm_mul_ps(c6_00
,FF
);
961 /* CUBIC SPLINE TABLE REPULSION */
962 vfitab
= _mm_add_epi32(vfitab
,ifour
);
963 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
964 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
965 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
966 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
967 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
968 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
969 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
970 fvdw12
= _mm_mul_ps(c12_00
,FF
);
971 fvdw
= _mm_xor_ps(signbit
,_mm_mul_ps(_mm_add_ps(fvdw6
,fvdw12
),_mm_mul_ps(vftabscale
,rinv00
)));
975 /* Update vectorial force */
976 fix0
= _mm_macc_ps(dx00
,fscal
,fix0
);
977 fiy0
= _mm_macc_ps(dy00
,fscal
,fiy0
);
978 fiz0
= _mm_macc_ps(dz00
,fscal
,fiz0
);
980 fjx0
= _mm_macc_ps(dx00
,fscal
,fjx0
);
981 fjy0
= _mm_macc_ps(dy00
,fscal
,fjy0
);
982 fjz0
= _mm_macc_ps(dz00
,fscal
,fjz0
);
984 /**************************
985 * CALCULATE INTERACTIONS *
986 **************************/
988 if (gmx_mm_any_lt(rsq10
,rcutoff2
))
991 /* Compute parameters for interactions between i and j atoms */
992 qq10
= _mm_mul_ps(iq1
,jq0
);
994 /* REACTION-FIELD ELECTROSTATICS */
995 felec
= _mm_mul_ps(qq10
,_mm_msub_ps(rinv10
,rinvsq10
,krf2
));
997 cutoff_mask
= _mm_cmplt_ps(rsq10
,rcutoff2
);
1001 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
1003 /* Update vectorial force */
1004 fix1
= _mm_macc_ps(dx10
,fscal
,fix1
);
1005 fiy1
= _mm_macc_ps(dy10
,fscal
,fiy1
);
1006 fiz1
= _mm_macc_ps(dz10
,fscal
,fiz1
);
1008 fjx0
= _mm_macc_ps(dx10
,fscal
,fjx0
);
1009 fjy0
= _mm_macc_ps(dy10
,fscal
,fjy0
);
1010 fjz0
= _mm_macc_ps(dz10
,fscal
,fjz0
);
1014 /**************************
1015 * CALCULATE INTERACTIONS *
1016 **************************/
1018 if (gmx_mm_any_lt(rsq20
,rcutoff2
))
1021 /* Compute parameters for interactions between i and j atoms */
1022 qq20
= _mm_mul_ps(iq2
,jq0
);
1024 /* REACTION-FIELD ELECTROSTATICS */
1025 felec
= _mm_mul_ps(qq20
,_mm_msub_ps(rinv20
,rinvsq20
,krf2
));
1027 cutoff_mask
= _mm_cmplt_ps(rsq20
,rcutoff2
);
1031 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
1033 /* Update vectorial force */
1034 fix2
= _mm_macc_ps(dx20
,fscal
,fix2
);
1035 fiy2
= _mm_macc_ps(dy20
,fscal
,fiy2
);
1036 fiz2
= _mm_macc_ps(dz20
,fscal
,fiz2
);
1038 fjx0
= _mm_macc_ps(dx20
,fscal
,fjx0
);
1039 fjy0
= _mm_macc_ps(dy20
,fscal
,fjy0
);
1040 fjz0
= _mm_macc_ps(dz20
,fscal
,fjz0
);
1044 /**************************
1045 * CALCULATE INTERACTIONS *
1046 **************************/
1048 if (gmx_mm_any_lt(rsq30
,rcutoff2
))
1051 /* Compute parameters for interactions between i and j atoms */
1052 qq30
= _mm_mul_ps(iq3
,jq0
);
1054 /* REACTION-FIELD ELECTROSTATICS */
1055 felec
= _mm_mul_ps(qq30
,_mm_msub_ps(rinv30
,rinvsq30
,krf2
));
1057 cutoff_mask
= _mm_cmplt_ps(rsq30
,rcutoff2
);
1061 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
1063 /* Update vectorial force */
1064 fix3
= _mm_macc_ps(dx30
,fscal
,fix3
);
1065 fiy3
= _mm_macc_ps(dy30
,fscal
,fiy3
);
1066 fiz3
= _mm_macc_ps(dz30
,fscal
,fiz3
);
1068 fjx0
= _mm_macc_ps(dx30
,fscal
,fjx0
);
1069 fjy0
= _mm_macc_ps(dy30
,fscal
,fjy0
);
1070 fjz0
= _mm_macc_ps(dz30
,fscal
,fjz0
);
1074 fjptrA
= f
+j_coord_offsetA
;
1075 fjptrB
= f
+j_coord_offsetB
;
1076 fjptrC
= f
+j_coord_offsetC
;
1077 fjptrD
= f
+j_coord_offsetD
;
1079 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjx0
,fjy0
,fjz0
);
1081 /* Inner loop uses 150 flops */
1084 if(jidx
<j_index_end
)
1087 /* Get j neighbor index, and coordinate index */
1088 jnrlistA
= jjnr
[jidx
];
1089 jnrlistB
= jjnr
[jidx
+1];
1090 jnrlistC
= jjnr
[jidx
+2];
1091 jnrlistD
= jjnr
[jidx
+3];
1092 /* Sign of each element will be negative for non-real atoms.
1093 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1094 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1096 dummy_mask
= gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
)),_mm_setzero_si128()));
1097 jnrA
= (jnrlistA
>=0) ? jnrlistA
: 0;
1098 jnrB
= (jnrlistB
>=0) ? jnrlistB
: 0;
1099 jnrC
= (jnrlistC
>=0) ? jnrlistC
: 0;
1100 jnrD
= (jnrlistD
>=0) ? jnrlistD
: 0;
1101 j_coord_offsetA
= DIM
*jnrA
;
1102 j_coord_offsetB
= DIM
*jnrB
;
1103 j_coord_offsetC
= DIM
*jnrC
;
1104 j_coord_offsetD
= DIM
*jnrD
;
1106 /* load j atom coordinates */
1107 gmx_mm_load_1rvec_4ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
1108 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
1111 /* Calculate displacement vector */
1112 dx00
= _mm_sub_ps(ix0
,jx0
);
1113 dy00
= _mm_sub_ps(iy0
,jy0
);
1114 dz00
= _mm_sub_ps(iz0
,jz0
);
1115 dx10
= _mm_sub_ps(ix1
,jx0
);
1116 dy10
= _mm_sub_ps(iy1
,jy0
);
1117 dz10
= _mm_sub_ps(iz1
,jz0
);
1118 dx20
= _mm_sub_ps(ix2
,jx0
);
1119 dy20
= _mm_sub_ps(iy2
,jy0
);
1120 dz20
= _mm_sub_ps(iz2
,jz0
);
1121 dx30
= _mm_sub_ps(ix3
,jx0
);
1122 dy30
= _mm_sub_ps(iy3
,jy0
);
1123 dz30
= _mm_sub_ps(iz3
,jz0
);
1125 /* Calculate squared distance and things based on it */
1126 rsq00
= gmx_mm_calc_rsq_ps(dx00
,dy00
,dz00
);
1127 rsq10
= gmx_mm_calc_rsq_ps(dx10
,dy10
,dz10
);
1128 rsq20
= gmx_mm_calc_rsq_ps(dx20
,dy20
,dz20
);
1129 rsq30
= gmx_mm_calc_rsq_ps(dx30
,dy30
,dz30
);
1131 rinv00
= gmx_mm_invsqrt_ps(rsq00
);
1132 rinv10
= gmx_mm_invsqrt_ps(rsq10
);
1133 rinv20
= gmx_mm_invsqrt_ps(rsq20
);
1134 rinv30
= gmx_mm_invsqrt_ps(rsq30
);
1136 rinvsq10
= _mm_mul_ps(rinv10
,rinv10
);
1137 rinvsq20
= _mm_mul_ps(rinv20
,rinv20
);
1138 rinvsq30
= _mm_mul_ps(rinv30
,rinv30
);
1140 /* Load parameters for j particles */
1141 jq0
= gmx_mm_load_4real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
1142 charge
+jnrC
+0,charge
+jnrD
+0);
1143 vdwjidx0A
= 2*vdwtype
[jnrA
+0];
1144 vdwjidx0B
= 2*vdwtype
[jnrB
+0];
1145 vdwjidx0C
= 2*vdwtype
[jnrC
+0];
1146 vdwjidx0D
= 2*vdwtype
[jnrD
+0];
1148 fjx0
= _mm_setzero_ps();
1149 fjy0
= _mm_setzero_ps();
1150 fjz0
= _mm_setzero_ps();
1152 /**************************
1153 * CALCULATE INTERACTIONS *
1154 **************************/
1156 r00
= _mm_mul_ps(rsq00
,rinv00
);
1157 r00
= _mm_andnot_ps(dummy_mask
,r00
);
1159 /* Compute parameters for interactions between i and j atoms */
1160 gmx_mm_load_4pair_swizzle_ps(vdwparam
+vdwioffset0
+vdwjidx0A
,
1161 vdwparam
+vdwioffset0
+vdwjidx0B
,
1162 vdwparam
+vdwioffset0
+vdwjidx0C
,
1163 vdwparam
+vdwioffset0
+vdwjidx0D
,
1166 /* Calculate table index by multiplying r with table scale and truncate to integer */
1167 rt
= _mm_mul_ps(r00
,vftabscale
);
1168 vfitab
= _mm_cvttps_epi32(rt
);
1170 vfeps
= _mm_frcz_ps(rt
);
1172 vfeps
= _mm_sub_ps(rt
,_mm_round_ps(rt
, _MM_FROUND_FLOOR
));
1174 twovfeps
= _mm_add_ps(vfeps
,vfeps
);
1175 vfitab
= _mm_slli_epi32(vfitab
,3);
1177 /* CUBIC SPLINE TABLE DISPERSION */
1178 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
1179 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
1180 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
1181 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
1182 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
1183 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
1184 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
1185 fvdw6
= _mm_mul_ps(c6_00
,FF
);
1187 /* CUBIC SPLINE TABLE REPULSION */
1188 vfitab
= _mm_add_epi32(vfitab
,ifour
);
1189 Y
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,0) );
1190 F
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,1) );
1191 G
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,2) );
1192 H
= _mm_load_ps( vftab
+ _mm_extract_epi32(vfitab
,3) );
1193 _MM_TRANSPOSE4_PS(Y
,F
,G
,H
);
1194 Fp
= _mm_macc_ps(vfeps
,_mm_macc_ps(H
,vfeps
,G
),F
);
1195 FF
= _mm_macc_ps(vfeps
,_mm_macc_ps(twovfeps
,H
,G
),Fp
);
1196 fvdw12
= _mm_mul_ps(c12_00
,FF
);
1197 fvdw
= _mm_xor_ps(signbit
,_mm_mul_ps(_mm_add_ps(fvdw6
,fvdw12
),_mm_mul_ps(vftabscale
,rinv00
)));
1201 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
1203 /* Update vectorial force */
1204 fix0
= _mm_macc_ps(dx00
,fscal
,fix0
);
1205 fiy0
= _mm_macc_ps(dy00
,fscal
,fiy0
);
1206 fiz0
= _mm_macc_ps(dz00
,fscal
,fiz0
);
1208 fjx0
= _mm_macc_ps(dx00
,fscal
,fjx0
);
1209 fjy0
= _mm_macc_ps(dy00
,fscal
,fjy0
);
1210 fjz0
= _mm_macc_ps(dz00
,fscal
,fjz0
);
1212 /**************************
1213 * CALCULATE INTERACTIONS *
1214 **************************/
1216 if (gmx_mm_any_lt(rsq10
,rcutoff2
))
1219 /* Compute parameters for interactions between i and j atoms */
1220 qq10
= _mm_mul_ps(iq1
,jq0
);
1222 /* REACTION-FIELD ELECTROSTATICS */
1223 felec
= _mm_mul_ps(qq10
,_mm_msub_ps(rinv10
,rinvsq10
,krf2
));
1225 cutoff_mask
= _mm_cmplt_ps(rsq10
,rcutoff2
);
1229 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
1231 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
1233 /* Update vectorial force */
1234 fix1
= _mm_macc_ps(dx10
,fscal
,fix1
);
1235 fiy1
= _mm_macc_ps(dy10
,fscal
,fiy1
);
1236 fiz1
= _mm_macc_ps(dz10
,fscal
,fiz1
);
1238 fjx0
= _mm_macc_ps(dx10
,fscal
,fjx0
);
1239 fjy0
= _mm_macc_ps(dy10
,fscal
,fjy0
);
1240 fjz0
= _mm_macc_ps(dz10
,fscal
,fjz0
);
1244 /**************************
1245 * CALCULATE INTERACTIONS *
1246 **************************/
1248 if (gmx_mm_any_lt(rsq20
,rcutoff2
))
1251 /* Compute parameters for interactions between i and j atoms */
1252 qq20
= _mm_mul_ps(iq2
,jq0
);
1254 /* REACTION-FIELD ELECTROSTATICS */
1255 felec
= _mm_mul_ps(qq20
,_mm_msub_ps(rinv20
,rinvsq20
,krf2
));
1257 cutoff_mask
= _mm_cmplt_ps(rsq20
,rcutoff2
);
1261 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
1263 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
1265 /* Update vectorial force */
1266 fix2
= _mm_macc_ps(dx20
,fscal
,fix2
);
1267 fiy2
= _mm_macc_ps(dy20
,fscal
,fiy2
);
1268 fiz2
= _mm_macc_ps(dz20
,fscal
,fiz2
);
1270 fjx0
= _mm_macc_ps(dx20
,fscal
,fjx0
);
1271 fjy0
= _mm_macc_ps(dy20
,fscal
,fjy0
);
1272 fjz0
= _mm_macc_ps(dz20
,fscal
,fjz0
);
1276 /**************************
1277 * CALCULATE INTERACTIONS *
1278 **************************/
1280 if (gmx_mm_any_lt(rsq30
,rcutoff2
))
1283 /* Compute parameters for interactions between i and j atoms */
1284 qq30
= _mm_mul_ps(iq3
,jq0
);
1286 /* REACTION-FIELD ELECTROSTATICS */
1287 felec
= _mm_mul_ps(qq30
,_mm_msub_ps(rinv30
,rinvsq30
,krf2
));
1289 cutoff_mask
= _mm_cmplt_ps(rsq30
,rcutoff2
);
1293 fscal
= _mm_and_ps(fscal
,cutoff_mask
);
1295 fscal
= _mm_andnot_ps(dummy_mask
,fscal
);
1297 /* Update vectorial force */
1298 fix3
= _mm_macc_ps(dx30
,fscal
,fix3
);
1299 fiy3
= _mm_macc_ps(dy30
,fscal
,fiy3
);
1300 fiz3
= _mm_macc_ps(dz30
,fscal
,fiz3
);
1302 fjx0
= _mm_macc_ps(dx30
,fscal
,fjx0
);
1303 fjy0
= _mm_macc_ps(dy30
,fscal
,fjy0
);
1304 fjz0
= _mm_macc_ps(dz30
,fscal
,fjz0
);
1308 fjptrA
= (jnrlistA
>=0) ? f
+j_coord_offsetA
: scratch
;
1309 fjptrB
= (jnrlistB
>=0) ? f
+j_coord_offsetB
: scratch
;
1310 fjptrC
= (jnrlistC
>=0) ? f
+j_coord_offsetC
: scratch
;
1311 fjptrD
= (jnrlistD
>=0) ? f
+j_coord_offsetD
: scratch
;
1313 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjx0
,fjy0
,fjz0
);
1315 /* Inner loop uses 151 flops */
1318 /* End of innermost loop */
1320 gmx_mm_update_iforce_4atom_swizzle_ps(fix0
,fiy0
,fiz0
,fix1
,fiy1
,fiz1
,fix2
,fiy2
,fiz2
,fix3
,fiy3
,fiz3
,
1321 f
+i_coord_offset
,fshift
+i_shift_offset
);
1323 /* Increment number of inner iterations */
1324 inneriter
+= j_index_end
- j_index_start
;
1326 /* Outer loop uses 24 flops */
1329 /* Increment number of outer iterations */
1332 /* Update outer/inner flops */
1334 inc_nrnb(nrnb
,eNR_NBKERNEL_ELEC_VDW_W4_F
,outeriter
*24 + inneriter
*151);