2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_256_single kernel generator.
42 #include "../nb_kernel.h"
43 #include "types/simple.h"
44 #include "gromacs/math/vec.h"
47 #include "gromacs/simd/math_x86_avx_256_single.h"
48 #include "kernelutil_x86_avx_256_single.h"
51 * Gromacs nonbonded kernel: nb_kernel_ElecEwSw_VdwNone_GeomW3P1_VF_avx_256_single
52 * Electrostatics interaction: Ewald
53 * VdW interaction: None
54 * Geometry: Water3-Particle
55 * Calculate force/pot: PotentialAndForce
58 nb_kernel_ElecEwSw_VdwNone_GeomW3P1_VF_avx_256_single
59 (t_nblist
* gmx_restrict nlist
,
60 rvec
* gmx_restrict xx
,
61 rvec
* gmx_restrict ff
,
62 t_forcerec
* gmx_restrict fr
,
63 t_mdatoms
* gmx_restrict mdatoms
,
64 nb_kernel_data_t gmx_unused
* gmx_restrict kernel_data
,
65 t_nrnb
* gmx_restrict nrnb
)
67 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
68 * just 0 for non-waters.
69 * Suffixes A,B,C,D,E,F,G,H refer to j loop unrolling done with AVX, e.g. for the eight different
70 * jnr indices corresponding to data put in the four positions in the SIMD register.
72 int i_shift_offset
,i_coord_offset
,outeriter
,inneriter
;
73 int j_index_start
,j_index_end
,jidx
,nri
,inr
,ggid
,iidx
;
74 int jnrA
,jnrB
,jnrC
,jnrD
;
75 int jnrE
,jnrF
,jnrG
,jnrH
;
76 int jnrlistA
,jnrlistB
,jnrlistC
,jnrlistD
;
77 int jnrlistE
,jnrlistF
,jnrlistG
,jnrlistH
;
78 int j_coord_offsetA
,j_coord_offsetB
,j_coord_offsetC
,j_coord_offsetD
;
79 int j_coord_offsetE
,j_coord_offsetF
,j_coord_offsetG
,j_coord_offsetH
;
80 int *iinr
,*jindex
,*jjnr
,*shiftidx
,*gid
;
82 real
*shiftvec
,*fshift
,*x
,*f
;
83 real
*fjptrA
,*fjptrB
,*fjptrC
,*fjptrD
,*fjptrE
,*fjptrF
,*fjptrG
,*fjptrH
;
85 __m256 tx
,ty
,tz
,fscal
,rcutoff
,rcutoff2
,jidxall
;
86 real
* vdwioffsetptr0
;
87 __m256 ix0
,iy0
,iz0
,fix0
,fiy0
,fiz0
,iq0
,isai0
;
88 real
* vdwioffsetptr1
;
89 __m256 ix1
,iy1
,iz1
,fix1
,fiy1
,fiz1
,iq1
,isai1
;
90 real
* vdwioffsetptr2
;
91 __m256 ix2
,iy2
,iz2
,fix2
,fiy2
,fiz2
,iq2
,isai2
;
92 int vdwjidx0A
,vdwjidx0B
,vdwjidx0C
,vdwjidx0D
,vdwjidx0E
,vdwjidx0F
,vdwjidx0G
,vdwjidx0H
;
93 __m256 jx0
,jy0
,jz0
,fjx0
,fjy0
,fjz0
,jq0
,isaj0
;
94 __m256 dx00
,dy00
,dz00
,rsq00
,rinv00
,rinvsq00
,r00
,qq00
,c6_00
,c12_00
;
95 __m256 dx10
,dy10
,dz10
,rsq10
,rinv10
,rinvsq10
,r10
,qq10
,c6_10
,c12_10
;
96 __m256 dx20
,dy20
,dz20
,rsq20
,rinv20
,rinvsq20
,r20
,qq20
,c6_20
,c12_20
;
97 __m256 velec
,felec
,velecsum
,facel
,crf
,krf
,krf2
;
100 __m128i ewitab_lo
,ewitab_hi
;
101 __m256 ewtabscale
,eweps
,sh_ewald
,ewrt
,ewtabhalfspace
,ewtabF
,ewtabFn
,ewtabD
,ewtabV
;
102 __m256 beta
,beta2
,beta3
,zeta2
,pmecorrF
,pmecorrV
,rinv3
;
104 __m256 rswitch
,swV3
,swV4
,swV5
,swF2
,swF3
,swF4
,d
,d2
,sw
,dsw
;
105 real rswitch_scalar
,d_scalar
;
106 __m256 dummy_mask
,cutoff_mask
;
107 __m256 signbit
= _mm256_castsi256_ps( _mm256_set1_epi32(0x80000000) );
108 __m256 one
= _mm256_set1_ps(1.0);
109 __m256 two
= _mm256_set1_ps(2.0);
115 jindex
= nlist
->jindex
;
117 shiftidx
= nlist
->shift
;
119 shiftvec
= fr
->shift_vec
[0];
120 fshift
= fr
->fshift
[0];
121 facel
= _mm256_set1_ps(fr
->epsfac
);
122 charge
= mdatoms
->chargeA
;
124 sh_ewald
= _mm256_set1_ps(fr
->ic
->sh_ewald
);
125 beta
= _mm256_set1_ps(fr
->ic
->ewaldcoeff_q
);
126 beta2
= _mm256_mul_ps(beta
,beta
);
127 beta3
= _mm256_mul_ps(beta
,beta2
);
129 ewtab
= fr
->ic
->tabq_coul_FDV0
;
130 ewtabscale
= _mm256_set1_ps(fr
->ic
->tabq_scale
);
131 ewtabhalfspace
= _mm256_set1_ps(0.5/fr
->ic
->tabq_scale
);
133 /* Setup water-specific parameters */
134 inr
= nlist
->iinr
[0];
135 iq0
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+0]));
136 iq1
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+1]));
137 iq2
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+2]));
139 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
140 rcutoff_scalar
= fr
->rcoulomb
;
141 rcutoff
= _mm256_set1_ps(rcutoff_scalar
);
142 rcutoff2
= _mm256_mul_ps(rcutoff
,rcutoff
);
144 rswitch_scalar
= fr
->rcoulomb_switch
;
145 rswitch
= _mm256_set1_ps(rswitch_scalar
);
146 /* Setup switch parameters */
147 d_scalar
= rcutoff_scalar
-rswitch_scalar
;
148 d
= _mm256_set1_ps(d_scalar
);
149 swV3
= _mm256_set1_ps(-10.0/(d_scalar
*d_scalar
*d_scalar
));
150 swV4
= _mm256_set1_ps( 15.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
));
151 swV5
= _mm256_set1_ps( -6.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
*d_scalar
));
152 swF2
= _mm256_set1_ps(-30.0/(d_scalar
*d_scalar
*d_scalar
));
153 swF3
= _mm256_set1_ps( 60.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
));
154 swF4
= _mm256_set1_ps(-30.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
*d_scalar
));
156 /* Avoid stupid compiler warnings */
157 jnrA
= jnrB
= jnrC
= jnrD
= jnrE
= jnrF
= jnrG
= jnrH
= 0;
170 for(iidx
=0;iidx
<4*DIM
;iidx
++)
175 /* Start outer loop over neighborlists */
176 for(iidx
=0; iidx
<nri
; iidx
++)
178 /* Load shift vector for this list */
179 i_shift_offset
= DIM
*shiftidx
[iidx
];
181 /* Load limits for loop over neighbors */
182 j_index_start
= jindex
[iidx
];
183 j_index_end
= jindex
[iidx
+1];
185 /* Get outer coordinate index */
187 i_coord_offset
= DIM
*inr
;
189 /* Load i particle coords and add shift vector */
190 gmx_mm256_load_shift_and_3rvec_broadcast_ps(shiftvec
+i_shift_offset
,x
+i_coord_offset
,
191 &ix0
,&iy0
,&iz0
,&ix1
,&iy1
,&iz1
,&ix2
,&iy2
,&iz2
);
193 fix0
= _mm256_setzero_ps();
194 fiy0
= _mm256_setzero_ps();
195 fiz0
= _mm256_setzero_ps();
196 fix1
= _mm256_setzero_ps();
197 fiy1
= _mm256_setzero_ps();
198 fiz1
= _mm256_setzero_ps();
199 fix2
= _mm256_setzero_ps();
200 fiy2
= _mm256_setzero_ps();
201 fiz2
= _mm256_setzero_ps();
203 /* Reset potential sums */
204 velecsum
= _mm256_setzero_ps();
206 /* Start inner kernel loop */
207 for(jidx
=j_index_start
; jidx
<j_index_end
&& jjnr
[jidx
+7]>=0; jidx
+=8)
210 /* Get j neighbor index, and coordinate index */
219 j_coord_offsetA
= DIM
*jnrA
;
220 j_coord_offsetB
= DIM
*jnrB
;
221 j_coord_offsetC
= DIM
*jnrC
;
222 j_coord_offsetD
= DIM
*jnrD
;
223 j_coord_offsetE
= DIM
*jnrE
;
224 j_coord_offsetF
= DIM
*jnrF
;
225 j_coord_offsetG
= DIM
*jnrG
;
226 j_coord_offsetH
= DIM
*jnrH
;
228 /* load j atom coordinates */
229 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
230 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
231 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
232 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
235 /* Calculate displacement vector */
236 dx00
= _mm256_sub_ps(ix0
,jx0
);
237 dy00
= _mm256_sub_ps(iy0
,jy0
);
238 dz00
= _mm256_sub_ps(iz0
,jz0
);
239 dx10
= _mm256_sub_ps(ix1
,jx0
);
240 dy10
= _mm256_sub_ps(iy1
,jy0
);
241 dz10
= _mm256_sub_ps(iz1
,jz0
);
242 dx20
= _mm256_sub_ps(ix2
,jx0
);
243 dy20
= _mm256_sub_ps(iy2
,jy0
);
244 dz20
= _mm256_sub_ps(iz2
,jz0
);
246 /* Calculate squared distance and things based on it */
247 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
248 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
249 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
251 rinv00
= gmx_mm256_invsqrt_ps(rsq00
);
252 rinv10
= gmx_mm256_invsqrt_ps(rsq10
);
253 rinv20
= gmx_mm256_invsqrt_ps(rsq20
);
255 rinvsq00
= _mm256_mul_ps(rinv00
,rinv00
);
256 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
257 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
259 /* Load parameters for j particles */
260 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
261 charge
+jnrC
+0,charge
+jnrD
+0,
262 charge
+jnrE
+0,charge
+jnrF
+0,
263 charge
+jnrG
+0,charge
+jnrH
+0);
265 fjx0
= _mm256_setzero_ps();
266 fjy0
= _mm256_setzero_ps();
267 fjz0
= _mm256_setzero_ps();
269 /**************************
270 * CALCULATE INTERACTIONS *
271 **************************/
273 if (gmx_mm256_any_lt(rsq00
,rcutoff2
))
276 r00
= _mm256_mul_ps(rsq00
,rinv00
);
278 /* Compute parameters for interactions between i and j atoms */
279 qq00
= _mm256_mul_ps(iq0
,jq0
);
281 /* EWALD ELECTROSTATICS */
283 /* Analytical PME correction */
284 zeta2
= _mm256_mul_ps(beta2
,rsq00
);
285 rinv3
= _mm256_mul_ps(rinvsq00
,rinv00
);
286 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
287 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
288 felec
= _mm256_mul_ps(qq00
,felec
);
289 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
290 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
291 velec
= _mm256_sub_ps(rinv00
,pmecorrV
);
292 velec
= _mm256_mul_ps(qq00
,velec
);
294 d
= _mm256_sub_ps(r00
,rswitch
);
295 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
296 d2
= _mm256_mul_ps(d
,d
);
297 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
299 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
301 /* Evaluate switch function */
302 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
303 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv00
,_mm256_mul_ps(velec
,dsw
)) );
304 velec
= _mm256_mul_ps(velec
,sw
);
305 cutoff_mask
= _mm256_cmp_ps(rsq00
,rcutoff2
,_CMP_LT_OQ
);
307 /* Update potential sum for this i atom from the interaction with this j atom. */
308 velec
= _mm256_and_ps(velec
,cutoff_mask
);
309 velecsum
= _mm256_add_ps(velecsum
,velec
);
313 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
315 /* Calculate temporary vectorial force */
316 tx
= _mm256_mul_ps(fscal
,dx00
);
317 ty
= _mm256_mul_ps(fscal
,dy00
);
318 tz
= _mm256_mul_ps(fscal
,dz00
);
320 /* Update vectorial force */
321 fix0
= _mm256_add_ps(fix0
,tx
);
322 fiy0
= _mm256_add_ps(fiy0
,ty
);
323 fiz0
= _mm256_add_ps(fiz0
,tz
);
325 fjx0
= _mm256_add_ps(fjx0
,tx
);
326 fjy0
= _mm256_add_ps(fjy0
,ty
);
327 fjz0
= _mm256_add_ps(fjz0
,tz
);
331 /**************************
332 * CALCULATE INTERACTIONS *
333 **************************/
335 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
338 r10
= _mm256_mul_ps(rsq10
,rinv10
);
340 /* Compute parameters for interactions between i and j atoms */
341 qq10
= _mm256_mul_ps(iq1
,jq0
);
343 /* EWALD ELECTROSTATICS */
345 /* Analytical PME correction */
346 zeta2
= _mm256_mul_ps(beta2
,rsq10
);
347 rinv3
= _mm256_mul_ps(rinvsq10
,rinv10
);
348 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
349 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
350 felec
= _mm256_mul_ps(qq10
,felec
);
351 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
352 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
353 velec
= _mm256_sub_ps(rinv10
,pmecorrV
);
354 velec
= _mm256_mul_ps(qq10
,velec
);
356 d
= _mm256_sub_ps(r10
,rswitch
);
357 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
358 d2
= _mm256_mul_ps(d
,d
);
359 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
361 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
363 /* Evaluate switch function */
364 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
365 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv10
,_mm256_mul_ps(velec
,dsw
)) );
366 velec
= _mm256_mul_ps(velec
,sw
);
367 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
369 /* Update potential sum for this i atom from the interaction with this j atom. */
370 velec
= _mm256_and_ps(velec
,cutoff_mask
);
371 velecsum
= _mm256_add_ps(velecsum
,velec
);
375 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
377 /* Calculate temporary vectorial force */
378 tx
= _mm256_mul_ps(fscal
,dx10
);
379 ty
= _mm256_mul_ps(fscal
,dy10
);
380 tz
= _mm256_mul_ps(fscal
,dz10
);
382 /* Update vectorial force */
383 fix1
= _mm256_add_ps(fix1
,tx
);
384 fiy1
= _mm256_add_ps(fiy1
,ty
);
385 fiz1
= _mm256_add_ps(fiz1
,tz
);
387 fjx0
= _mm256_add_ps(fjx0
,tx
);
388 fjy0
= _mm256_add_ps(fjy0
,ty
);
389 fjz0
= _mm256_add_ps(fjz0
,tz
);
393 /**************************
394 * CALCULATE INTERACTIONS *
395 **************************/
397 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
400 r20
= _mm256_mul_ps(rsq20
,rinv20
);
402 /* Compute parameters for interactions between i and j atoms */
403 qq20
= _mm256_mul_ps(iq2
,jq0
);
405 /* EWALD ELECTROSTATICS */
407 /* Analytical PME correction */
408 zeta2
= _mm256_mul_ps(beta2
,rsq20
);
409 rinv3
= _mm256_mul_ps(rinvsq20
,rinv20
);
410 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
411 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
412 felec
= _mm256_mul_ps(qq20
,felec
);
413 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
414 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
415 velec
= _mm256_sub_ps(rinv20
,pmecorrV
);
416 velec
= _mm256_mul_ps(qq20
,velec
);
418 d
= _mm256_sub_ps(r20
,rswitch
);
419 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
420 d2
= _mm256_mul_ps(d
,d
);
421 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
423 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
425 /* Evaluate switch function */
426 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
427 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv20
,_mm256_mul_ps(velec
,dsw
)) );
428 velec
= _mm256_mul_ps(velec
,sw
);
429 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
431 /* Update potential sum for this i atom from the interaction with this j atom. */
432 velec
= _mm256_and_ps(velec
,cutoff_mask
);
433 velecsum
= _mm256_add_ps(velecsum
,velec
);
437 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
439 /* Calculate temporary vectorial force */
440 tx
= _mm256_mul_ps(fscal
,dx20
);
441 ty
= _mm256_mul_ps(fscal
,dy20
);
442 tz
= _mm256_mul_ps(fscal
,dz20
);
444 /* Update vectorial force */
445 fix2
= _mm256_add_ps(fix2
,tx
);
446 fiy2
= _mm256_add_ps(fiy2
,ty
);
447 fiz2
= _mm256_add_ps(fiz2
,tz
);
449 fjx0
= _mm256_add_ps(fjx0
,tx
);
450 fjy0
= _mm256_add_ps(fjy0
,ty
);
451 fjz0
= _mm256_add_ps(fjz0
,tz
);
455 fjptrA
= f
+j_coord_offsetA
;
456 fjptrB
= f
+j_coord_offsetB
;
457 fjptrC
= f
+j_coord_offsetC
;
458 fjptrD
= f
+j_coord_offsetD
;
459 fjptrE
= f
+j_coord_offsetE
;
460 fjptrF
= f
+j_coord_offsetF
;
461 fjptrG
= f
+j_coord_offsetG
;
462 fjptrH
= f
+j_coord_offsetH
;
464 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
466 /* Inner loop uses 327 flops */
472 /* Get j neighbor index, and coordinate index */
473 jnrlistA
= jjnr
[jidx
];
474 jnrlistB
= jjnr
[jidx
+1];
475 jnrlistC
= jjnr
[jidx
+2];
476 jnrlistD
= jjnr
[jidx
+3];
477 jnrlistE
= jjnr
[jidx
+4];
478 jnrlistF
= jjnr
[jidx
+5];
479 jnrlistG
= jjnr
[jidx
+6];
480 jnrlistH
= jjnr
[jidx
+7];
481 /* Sign of each element will be negative for non-real atoms.
482 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
483 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
485 dummy_mask
= gmx_mm256_set_m128(gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
+4)),_mm_setzero_si128())),
486 gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
)),_mm_setzero_si128())));
488 jnrA
= (jnrlistA
>=0) ? jnrlistA
: 0;
489 jnrB
= (jnrlistB
>=0) ? jnrlistB
: 0;
490 jnrC
= (jnrlistC
>=0) ? jnrlistC
: 0;
491 jnrD
= (jnrlistD
>=0) ? jnrlistD
: 0;
492 jnrE
= (jnrlistE
>=0) ? jnrlistE
: 0;
493 jnrF
= (jnrlistF
>=0) ? jnrlistF
: 0;
494 jnrG
= (jnrlistG
>=0) ? jnrlistG
: 0;
495 jnrH
= (jnrlistH
>=0) ? jnrlistH
: 0;
496 j_coord_offsetA
= DIM
*jnrA
;
497 j_coord_offsetB
= DIM
*jnrB
;
498 j_coord_offsetC
= DIM
*jnrC
;
499 j_coord_offsetD
= DIM
*jnrD
;
500 j_coord_offsetE
= DIM
*jnrE
;
501 j_coord_offsetF
= DIM
*jnrF
;
502 j_coord_offsetG
= DIM
*jnrG
;
503 j_coord_offsetH
= DIM
*jnrH
;
505 /* load j atom coordinates */
506 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
507 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
508 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
509 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
512 /* Calculate displacement vector */
513 dx00
= _mm256_sub_ps(ix0
,jx0
);
514 dy00
= _mm256_sub_ps(iy0
,jy0
);
515 dz00
= _mm256_sub_ps(iz0
,jz0
);
516 dx10
= _mm256_sub_ps(ix1
,jx0
);
517 dy10
= _mm256_sub_ps(iy1
,jy0
);
518 dz10
= _mm256_sub_ps(iz1
,jz0
);
519 dx20
= _mm256_sub_ps(ix2
,jx0
);
520 dy20
= _mm256_sub_ps(iy2
,jy0
);
521 dz20
= _mm256_sub_ps(iz2
,jz0
);
523 /* Calculate squared distance and things based on it */
524 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
525 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
526 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
528 rinv00
= gmx_mm256_invsqrt_ps(rsq00
);
529 rinv10
= gmx_mm256_invsqrt_ps(rsq10
);
530 rinv20
= gmx_mm256_invsqrt_ps(rsq20
);
532 rinvsq00
= _mm256_mul_ps(rinv00
,rinv00
);
533 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
534 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
536 /* Load parameters for j particles */
537 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
538 charge
+jnrC
+0,charge
+jnrD
+0,
539 charge
+jnrE
+0,charge
+jnrF
+0,
540 charge
+jnrG
+0,charge
+jnrH
+0);
542 fjx0
= _mm256_setzero_ps();
543 fjy0
= _mm256_setzero_ps();
544 fjz0
= _mm256_setzero_ps();
546 /**************************
547 * CALCULATE INTERACTIONS *
548 **************************/
550 if (gmx_mm256_any_lt(rsq00
,rcutoff2
))
553 r00
= _mm256_mul_ps(rsq00
,rinv00
);
554 r00
= _mm256_andnot_ps(dummy_mask
,r00
);
556 /* Compute parameters for interactions between i and j atoms */
557 qq00
= _mm256_mul_ps(iq0
,jq0
);
559 /* EWALD ELECTROSTATICS */
561 /* Analytical PME correction */
562 zeta2
= _mm256_mul_ps(beta2
,rsq00
);
563 rinv3
= _mm256_mul_ps(rinvsq00
,rinv00
);
564 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
565 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
566 felec
= _mm256_mul_ps(qq00
,felec
);
567 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
568 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
569 velec
= _mm256_sub_ps(rinv00
,pmecorrV
);
570 velec
= _mm256_mul_ps(qq00
,velec
);
572 d
= _mm256_sub_ps(r00
,rswitch
);
573 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
574 d2
= _mm256_mul_ps(d
,d
);
575 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
577 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
579 /* Evaluate switch function */
580 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
581 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv00
,_mm256_mul_ps(velec
,dsw
)) );
582 velec
= _mm256_mul_ps(velec
,sw
);
583 cutoff_mask
= _mm256_cmp_ps(rsq00
,rcutoff2
,_CMP_LT_OQ
);
585 /* Update potential sum for this i atom from the interaction with this j atom. */
586 velec
= _mm256_and_ps(velec
,cutoff_mask
);
587 velec
= _mm256_andnot_ps(dummy_mask
,velec
);
588 velecsum
= _mm256_add_ps(velecsum
,velec
);
592 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
594 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
596 /* Calculate temporary vectorial force */
597 tx
= _mm256_mul_ps(fscal
,dx00
);
598 ty
= _mm256_mul_ps(fscal
,dy00
);
599 tz
= _mm256_mul_ps(fscal
,dz00
);
601 /* Update vectorial force */
602 fix0
= _mm256_add_ps(fix0
,tx
);
603 fiy0
= _mm256_add_ps(fiy0
,ty
);
604 fiz0
= _mm256_add_ps(fiz0
,tz
);
606 fjx0
= _mm256_add_ps(fjx0
,tx
);
607 fjy0
= _mm256_add_ps(fjy0
,ty
);
608 fjz0
= _mm256_add_ps(fjz0
,tz
);
612 /**************************
613 * CALCULATE INTERACTIONS *
614 **************************/
616 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
619 r10
= _mm256_mul_ps(rsq10
,rinv10
);
620 r10
= _mm256_andnot_ps(dummy_mask
,r10
);
622 /* Compute parameters for interactions between i and j atoms */
623 qq10
= _mm256_mul_ps(iq1
,jq0
);
625 /* EWALD ELECTROSTATICS */
627 /* Analytical PME correction */
628 zeta2
= _mm256_mul_ps(beta2
,rsq10
);
629 rinv3
= _mm256_mul_ps(rinvsq10
,rinv10
);
630 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
631 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
632 felec
= _mm256_mul_ps(qq10
,felec
);
633 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
634 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
635 velec
= _mm256_sub_ps(rinv10
,pmecorrV
);
636 velec
= _mm256_mul_ps(qq10
,velec
);
638 d
= _mm256_sub_ps(r10
,rswitch
);
639 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
640 d2
= _mm256_mul_ps(d
,d
);
641 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
643 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
645 /* Evaluate switch function */
646 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
647 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv10
,_mm256_mul_ps(velec
,dsw
)) );
648 velec
= _mm256_mul_ps(velec
,sw
);
649 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
651 /* Update potential sum for this i atom from the interaction with this j atom. */
652 velec
= _mm256_and_ps(velec
,cutoff_mask
);
653 velec
= _mm256_andnot_ps(dummy_mask
,velec
);
654 velecsum
= _mm256_add_ps(velecsum
,velec
);
658 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
660 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
662 /* Calculate temporary vectorial force */
663 tx
= _mm256_mul_ps(fscal
,dx10
);
664 ty
= _mm256_mul_ps(fscal
,dy10
);
665 tz
= _mm256_mul_ps(fscal
,dz10
);
667 /* Update vectorial force */
668 fix1
= _mm256_add_ps(fix1
,tx
);
669 fiy1
= _mm256_add_ps(fiy1
,ty
);
670 fiz1
= _mm256_add_ps(fiz1
,tz
);
672 fjx0
= _mm256_add_ps(fjx0
,tx
);
673 fjy0
= _mm256_add_ps(fjy0
,ty
);
674 fjz0
= _mm256_add_ps(fjz0
,tz
);
678 /**************************
679 * CALCULATE INTERACTIONS *
680 **************************/
682 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
685 r20
= _mm256_mul_ps(rsq20
,rinv20
);
686 r20
= _mm256_andnot_ps(dummy_mask
,r20
);
688 /* Compute parameters for interactions between i and j atoms */
689 qq20
= _mm256_mul_ps(iq2
,jq0
);
691 /* EWALD ELECTROSTATICS */
693 /* Analytical PME correction */
694 zeta2
= _mm256_mul_ps(beta2
,rsq20
);
695 rinv3
= _mm256_mul_ps(rinvsq20
,rinv20
);
696 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
697 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
698 felec
= _mm256_mul_ps(qq20
,felec
);
699 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
700 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
701 velec
= _mm256_sub_ps(rinv20
,pmecorrV
);
702 velec
= _mm256_mul_ps(qq20
,velec
);
704 d
= _mm256_sub_ps(r20
,rswitch
);
705 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
706 d2
= _mm256_mul_ps(d
,d
);
707 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
709 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
711 /* Evaluate switch function */
712 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
713 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv20
,_mm256_mul_ps(velec
,dsw
)) );
714 velec
= _mm256_mul_ps(velec
,sw
);
715 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
717 /* Update potential sum for this i atom from the interaction with this j atom. */
718 velec
= _mm256_and_ps(velec
,cutoff_mask
);
719 velec
= _mm256_andnot_ps(dummy_mask
,velec
);
720 velecsum
= _mm256_add_ps(velecsum
,velec
);
724 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
726 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
728 /* Calculate temporary vectorial force */
729 tx
= _mm256_mul_ps(fscal
,dx20
);
730 ty
= _mm256_mul_ps(fscal
,dy20
);
731 tz
= _mm256_mul_ps(fscal
,dz20
);
733 /* Update vectorial force */
734 fix2
= _mm256_add_ps(fix2
,tx
);
735 fiy2
= _mm256_add_ps(fiy2
,ty
);
736 fiz2
= _mm256_add_ps(fiz2
,tz
);
738 fjx0
= _mm256_add_ps(fjx0
,tx
);
739 fjy0
= _mm256_add_ps(fjy0
,ty
);
740 fjz0
= _mm256_add_ps(fjz0
,tz
);
744 fjptrA
= (jnrlistA
>=0) ? f
+j_coord_offsetA
: scratch
;
745 fjptrB
= (jnrlistB
>=0) ? f
+j_coord_offsetB
: scratch
;
746 fjptrC
= (jnrlistC
>=0) ? f
+j_coord_offsetC
: scratch
;
747 fjptrD
= (jnrlistD
>=0) ? f
+j_coord_offsetD
: scratch
;
748 fjptrE
= (jnrlistE
>=0) ? f
+j_coord_offsetE
: scratch
;
749 fjptrF
= (jnrlistF
>=0) ? f
+j_coord_offsetF
: scratch
;
750 fjptrG
= (jnrlistG
>=0) ? f
+j_coord_offsetG
: scratch
;
751 fjptrH
= (jnrlistH
>=0) ? f
+j_coord_offsetH
: scratch
;
753 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
755 /* Inner loop uses 330 flops */
758 /* End of innermost loop */
760 gmx_mm256_update_iforce_3atom_swizzle_ps(fix0
,fiy0
,fiz0
,fix1
,fiy1
,fiz1
,fix2
,fiy2
,fiz2
,
761 f
+i_coord_offset
,fshift
+i_shift_offset
);
764 /* Update potential energies */
765 gmx_mm256_update_1pot_ps(velecsum
,kernel_data
->energygrp_elec
+ggid
);
767 /* Increment number of inner iterations */
768 inneriter
+= j_index_end
- j_index_start
;
770 /* Outer loop uses 19 flops */
773 /* Increment number of outer iterations */
776 /* Update outer/inner flops */
778 inc_nrnb(nrnb
,eNR_NBKERNEL_ELEC_W3_VF
,outeriter
*19 + inneriter
*330);
781 * Gromacs nonbonded kernel: nb_kernel_ElecEwSw_VdwNone_GeomW3P1_F_avx_256_single
782 * Electrostatics interaction: Ewald
783 * VdW interaction: None
784 * Geometry: Water3-Particle
785 * Calculate force/pot: Force
788 nb_kernel_ElecEwSw_VdwNone_GeomW3P1_F_avx_256_single
789 (t_nblist
* gmx_restrict nlist
,
790 rvec
* gmx_restrict xx
,
791 rvec
* gmx_restrict ff
,
792 t_forcerec
* gmx_restrict fr
,
793 t_mdatoms
* gmx_restrict mdatoms
,
794 nb_kernel_data_t gmx_unused
* gmx_restrict kernel_data
,
795 t_nrnb
* gmx_restrict nrnb
)
797 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
798 * just 0 for non-waters.
799 * Suffixes A,B,C,D,E,F,G,H refer to j loop unrolling done with AVX, e.g. for the eight different
800 * jnr indices corresponding to data put in the four positions in the SIMD register.
802 int i_shift_offset
,i_coord_offset
,outeriter
,inneriter
;
803 int j_index_start
,j_index_end
,jidx
,nri
,inr
,ggid
,iidx
;
804 int jnrA
,jnrB
,jnrC
,jnrD
;
805 int jnrE
,jnrF
,jnrG
,jnrH
;
806 int jnrlistA
,jnrlistB
,jnrlistC
,jnrlistD
;
807 int jnrlistE
,jnrlistF
,jnrlistG
,jnrlistH
;
808 int j_coord_offsetA
,j_coord_offsetB
,j_coord_offsetC
,j_coord_offsetD
;
809 int j_coord_offsetE
,j_coord_offsetF
,j_coord_offsetG
,j_coord_offsetH
;
810 int *iinr
,*jindex
,*jjnr
,*shiftidx
,*gid
;
812 real
*shiftvec
,*fshift
,*x
,*f
;
813 real
*fjptrA
,*fjptrB
,*fjptrC
,*fjptrD
,*fjptrE
,*fjptrF
,*fjptrG
,*fjptrH
;
815 __m256 tx
,ty
,tz
,fscal
,rcutoff
,rcutoff2
,jidxall
;
816 real
* vdwioffsetptr0
;
817 __m256 ix0
,iy0
,iz0
,fix0
,fiy0
,fiz0
,iq0
,isai0
;
818 real
* vdwioffsetptr1
;
819 __m256 ix1
,iy1
,iz1
,fix1
,fiy1
,fiz1
,iq1
,isai1
;
820 real
* vdwioffsetptr2
;
821 __m256 ix2
,iy2
,iz2
,fix2
,fiy2
,fiz2
,iq2
,isai2
;
822 int vdwjidx0A
,vdwjidx0B
,vdwjidx0C
,vdwjidx0D
,vdwjidx0E
,vdwjidx0F
,vdwjidx0G
,vdwjidx0H
;
823 __m256 jx0
,jy0
,jz0
,fjx0
,fjy0
,fjz0
,jq0
,isaj0
;
824 __m256 dx00
,dy00
,dz00
,rsq00
,rinv00
,rinvsq00
,r00
,qq00
,c6_00
,c12_00
;
825 __m256 dx10
,dy10
,dz10
,rsq10
,rinv10
,rinvsq10
,r10
,qq10
,c6_10
,c12_10
;
826 __m256 dx20
,dy20
,dz20
,rsq20
,rinv20
,rinvsq20
,r20
,qq20
,c6_20
,c12_20
;
827 __m256 velec
,felec
,velecsum
,facel
,crf
,krf
,krf2
;
830 __m128i ewitab_lo
,ewitab_hi
;
831 __m256 ewtabscale
,eweps
,sh_ewald
,ewrt
,ewtabhalfspace
,ewtabF
,ewtabFn
,ewtabD
,ewtabV
;
832 __m256 beta
,beta2
,beta3
,zeta2
,pmecorrF
,pmecorrV
,rinv3
;
834 __m256 rswitch
,swV3
,swV4
,swV5
,swF2
,swF3
,swF4
,d
,d2
,sw
,dsw
;
835 real rswitch_scalar
,d_scalar
;
836 __m256 dummy_mask
,cutoff_mask
;
837 __m256 signbit
= _mm256_castsi256_ps( _mm256_set1_epi32(0x80000000) );
838 __m256 one
= _mm256_set1_ps(1.0);
839 __m256 two
= _mm256_set1_ps(2.0);
845 jindex
= nlist
->jindex
;
847 shiftidx
= nlist
->shift
;
849 shiftvec
= fr
->shift_vec
[0];
850 fshift
= fr
->fshift
[0];
851 facel
= _mm256_set1_ps(fr
->epsfac
);
852 charge
= mdatoms
->chargeA
;
854 sh_ewald
= _mm256_set1_ps(fr
->ic
->sh_ewald
);
855 beta
= _mm256_set1_ps(fr
->ic
->ewaldcoeff_q
);
856 beta2
= _mm256_mul_ps(beta
,beta
);
857 beta3
= _mm256_mul_ps(beta
,beta2
);
859 ewtab
= fr
->ic
->tabq_coul_FDV0
;
860 ewtabscale
= _mm256_set1_ps(fr
->ic
->tabq_scale
);
861 ewtabhalfspace
= _mm256_set1_ps(0.5/fr
->ic
->tabq_scale
);
863 /* Setup water-specific parameters */
864 inr
= nlist
->iinr
[0];
865 iq0
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+0]));
866 iq1
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+1]));
867 iq2
= _mm256_mul_ps(facel
,_mm256_set1_ps(charge
[inr
+2]));
869 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
870 rcutoff_scalar
= fr
->rcoulomb
;
871 rcutoff
= _mm256_set1_ps(rcutoff_scalar
);
872 rcutoff2
= _mm256_mul_ps(rcutoff
,rcutoff
);
874 rswitch_scalar
= fr
->rcoulomb_switch
;
875 rswitch
= _mm256_set1_ps(rswitch_scalar
);
876 /* Setup switch parameters */
877 d_scalar
= rcutoff_scalar
-rswitch_scalar
;
878 d
= _mm256_set1_ps(d_scalar
);
879 swV3
= _mm256_set1_ps(-10.0/(d_scalar
*d_scalar
*d_scalar
));
880 swV4
= _mm256_set1_ps( 15.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
));
881 swV5
= _mm256_set1_ps( -6.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
*d_scalar
));
882 swF2
= _mm256_set1_ps(-30.0/(d_scalar
*d_scalar
*d_scalar
));
883 swF3
= _mm256_set1_ps( 60.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
));
884 swF4
= _mm256_set1_ps(-30.0/(d_scalar
*d_scalar
*d_scalar
*d_scalar
*d_scalar
));
886 /* Avoid stupid compiler warnings */
887 jnrA
= jnrB
= jnrC
= jnrD
= jnrE
= jnrF
= jnrG
= jnrH
= 0;
900 for(iidx
=0;iidx
<4*DIM
;iidx
++)
905 /* Start outer loop over neighborlists */
906 for(iidx
=0; iidx
<nri
; iidx
++)
908 /* Load shift vector for this list */
909 i_shift_offset
= DIM
*shiftidx
[iidx
];
911 /* Load limits for loop over neighbors */
912 j_index_start
= jindex
[iidx
];
913 j_index_end
= jindex
[iidx
+1];
915 /* Get outer coordinate index */
917 i_coord_offset
= DIM
*inr
;
919 /* Load i particle coords and add shift vector */
920 gmx_mm256_load_shift_and_3rvec_broadcast_ps(shiftvec
+i_shift_offset
,x
+i_coord_offset
,
921 &ix0
,&iy0
,&iz0
,&ix1
,&iy1
,&iz1
,&ix2
,&iy2
,&iz2
);
923 fix0
= _mm256_setzero_ps();
924 fiy0
= _mm256_setzero_ps();
925 fiz0
= _mm256_setzero_ps();
926 fix1
= _mm256_setzero_ps();
927 fiy1
= _mm256_setzero_ps();
928 fiz1
= _mm256_setzero_ps();
929 fix2
= _mm256_setzero_ps();
930 fiy2
= _mm256_setzero_ps();
931 fiz2
= _mm256_setzero_ps();
933 /* Start inner kernel loop */
934 for(jidx
=j_index_start
; jidx
<j_index_end
&& jjnr
[jidx
+7]>=0; jidx
+=8)
937 /* Get j neighbor index, and coordinate index */
946 j_coord_offsetA
= DIM
*jnrA
;
947 j_coord_offsetB
= DIM
*jnrB
;
948 j_coord_offsetC
= DIM
*jnrC
;
949 j_coord_offsetD
= DIM
*jnrD
;
950 j_coord_offsetE
= DIM
*jnrE
;
951 j_coord_offsetF
= DIM
*jnrF
;
952 j_coord_offsetG
= DIM
*jnrG
;
953 j_coord_offsetH
= DIM
*jnrH
;
955 /* load j atom coordinates */
956 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
957 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
958 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
959 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
962 /* Calculate displacement vector */
963 dx00
= _mm256_sub_ps(ix0
,jx0
);
964 dy00
= _mm256_sub_ps(iy0
,jy0
);
965 dz00
= _mm256_sub_ps(iz0
,jz0
);
966 dx10
= _mm256_sub_ps(ix1
,jx0
);
967 dy10
= _mm256_sub_ps(iy1
,jy0
);
968 dz10
= _mm256_sub_ps(iz1
,jz0
);
969 dx20
= _mm256_sub_ps(ix2
,jx0
);
970 dy20
= _mm256_sub_ps(iy2
,jy0
);
971 dz20
= _mm256_sub_ps(iz2
,jz0
);
973 /* Calculate squared distance and things based on it */
974 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
975 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
976 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
978 rinv00
= gmx_mm256_invsqrt_ps(rsq00
);
979 rinv10
= gmx_mm256_invsqrt_ps(rsq10
);
980 rinv20
= gmx_mm256_invsqrt_ps(rsq20
);
982 rinvsq00
= _mm256_mul_ps(rinv00
,rinv00
);
983 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
984 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
986 /* Load parameters for j particles */
987 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
988 charge
+jnrC
+0,charge
+jnrD
+0,
989 charge
+jnrE
+0,charge
+jnrF
+0,
990 charge
+jnrG
+0,charge
+jnrH
+0);
992 fjx0
= _mm256_setzero_ps();
993 fjy0
= _mm256_setzero_ps();
994 fjz0
= _mm256_setzero_ps();
996 /**************************
997 * CALCULATE INTERACTIONS *
998 **************************/
1000 if (gmx_mm256_any_lt(rsq00
,rcutoff2
))
1003 r00
= _mm256_mul_ps(rsq00
,rinv00
);
1005 /* Compute parameters for interactions between i and j atoms */
1006 qq00
= _mm256_mul_ps(iq0
,jq0
);
1008 /* EWALD ELECTROSTATICS */
1010 /* Analytical PME correction */
1011 zeta2
= _mm256_mul_ps(beta2
,rsq00
);
1012 rinv3
= _mm256_mul_ps(rinvsq00
,rinv00
);
1013 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
1014 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
1015 felec
= _mm256_mul_ps(qq00
,felec
);
1016 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
1017 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
1018 velec
= _mm256_sub_ps(rinv00
,pmecorrV
);
1019 velec
= _mm256_mul_ps(qq00
,velec
);
1021 d
= _mm256_sub_ps(r00
,rswitch
);
1022 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
1023 d2
= _mm256_mul_ps(d
,d
);
1024 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
1026 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
1028 /* Evaluate switch function */
1029 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1030 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv00
,_mm256_mul_ps(velec
,dsw
)) );
1031 cutoff_mask
= _mm256_cmp_ps(rsq00
,rcutoff2
,_CMP_LT_OQ
);
1035 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1037 /* Calculate temporary vectorial force */
1038 tx
= _mm256_mul_ps(fscal
,dx00
);
1039 ty
= _mm256_mul_ps(fscal
,dy00
);
1040 tz
= _mm256_mul_ps(fscal
,dz00
);
1042 /* Update vectorial force */
1043 fix0
= _mm256_add_ps(fix0
,tx
);
1044 fiy0
= _mm256_add_ps(fiy0
,ty
);
1045 fiz0
= _mm256_add_ps(fiz0
,tz
);
1047 fjx0
= _mm256_add_ps(fjx0
,tx
);
1048 fjy0
= _mm256_add_ps(fjy0
,ty
);
1049 fjz0
= _mm256_add_ps(fjz0
,tz
);
1053 /**************************
1054 * CALCULATE INTERACTIONS *
1055 **************************/
1057 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
1060 r10
= _mm256_mul_ps(rsq10
,rinv10
);
1062 /* Compute parameters for interactions between i and j atoms */
1063 qq10
= _mm256_mul_ps(iq1
,jq0
);
1065 /* EWALD ELECTROSTATICS */
1067 /* Analytical PME correction */
1068 zeta2
= _mm256_mul_ps(beta2
,rsq10
);
1069 rinv3
= _mm256_mul_ps(rinvsq10
,rinv10
);
1070 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
1071 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
1072 felec
= _mm256_mul_ps(qq10
,felec
);
1073 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
1074 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
1075 velec
= _mm256_sub_ps(rinv10
,pmecorrV
);
1076 velec
= _mm256_mul_ps(qq10
,velec
);
1078 d
= _mm256_sub_ps(r10
,rswitch
);
1079 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
1080 d2
= _mm256_mul_ps(d
,d
);
1081 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
1083 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
1085 /* Evaluate switch function */
1086 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1087 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv10
,_mm256_mul_ps(velec
,dsw
)) );
1088 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
1092 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1094 /* Calculate temporary vectorial force */
1095 tx
= _mm256_mul_ps(fscal
,dx10
);
1096 ty
= _mm256_mul_ps(fscal
,dy10
);
1097 tz
= _mm256_mul_ps(fscal
,dz10
);
1099 /* Update vectorial force */
1100 fix1
= _mm256_add_ps(fix1
,tx
);
1101 fiy1
= _mm256_add_ps(fiy1
,ty
);
1102 fiz1
= _mm256_add_ps(fiz1
,tz
);
1104 fjx0
= _mm256_add_ps(fjx0
,tx
);
1105 fjy0
= _mm256_add_ps(fjy0
,ty
);
1106 fjz0
= _mm256_add_ps(fjz0
,tz
);
1110 /**************************
1111 * CALCULATE INTERACTIONS *
1112 **************************/
1114 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
1117 r20
= _mm256_mul_ps(rsq20
,rinv20
);
1119 /* Compute parameters for interactions between i and j atoms */
1120 qq20
= _mm256_mul_ps(iq2
,jq0
);
1122 /* EWALD ELECTROSTATICS */
1124 /* Analytical PME correction */
1125 zeta2
= _mm256_mul_ps(beta2
,rsq20
);
1126 rinv3
= _mm256_mul_ps(rinvsq20
,rinv20
);
1127 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
1128 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
1129 felec
= _mm256_mul_ps(qq20
,felec
);
1130 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
1131 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
1132 velec
= _mm256_sub_ps(rinv20
,pmecorrV
);
1133 velec
= _mm256_mul_ps(qq20
,velec
);
1135 d
= _mm256_sub_ps(r20
,rswitch
);
1136 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
1137 d2
= _mm256_mul_ps(d
,d
);
1138 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
1140 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
1142 /* Evaluate switch function */
1143 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1144 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv20
,_mm256_mul_ps(velec
,dsw
)) );
1145 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
1149 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1151 /* Calculate temporary vectorial force */
1152 tx
= _mm256_mul_ps(fscal
,dx20
);
1153 ty
= _mm256_mul_ps(fscal
,dy20
);
1154 tz
= _mm256_mul_ps(fscal
,dz20
);
1156 /* Update vectorial force */
1157 fix2
= _mm256_add_ps(fix2
,tx
);
1158 fiy2
= _mm256_add_ps(fiy2
,ty
);
1159 fiz2
= _mm256_add_ps(fiz2
,tz
);
1161 fjx0
= _mm256_add_ps(fjx0
,tx
);
1162 fjy0
= _mm256_add_ps(fjy0
,ty
);
1163 fjz0
= _mm256_add_ps(fjz0
,tz
);
1167 fjptrA
= f
+j_coord_offsetA
;
1168 fjptrB
= f
+j_coord_offsetB
;
1169 fjptrC
= f
+j_coord_offsetC
;
1170 fjptrD
= f
+j_coord_offsetD
;
1171 fjptrE
= f
+j_coord_offsetE
;
1172 fjptrF
= f
+j_coord_offsetF
;
1173 fjptrG
= f
+j_coord_offsetG
;
1174 fjptrH
= f
+j_coord_offsetH
;
1176 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
1178 /* Inner loop uses 318 flops */
1181 if(jidx
<j_index_end
)
1184 /* Get j neighbor index, and coordinate index */
1185 jnrlistA
= jjnr
[jidx
];
1186 jnrlistB
= jjnr
[jidx
+1];
1187 jnrlistC
= jjnr
[jidx
+2];
1188 jnrlistD
= jjnr
[jidx
+3];
1189 jnrlistE
= jjnr
[jidx
+4];
1190 jnrlistF
= jjnr
[jidx
+5];
1191 jnrlistG
= jjnr
[jidx
+6];
1192 jnrlistH
= jjnr
[jidx
+7];
1193 /* Sign of each element will be negative for non-real atoms.
1194 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1195 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1197 dummy_mask
= gmx_mm256_set_m128(gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
+4)),_mm_setzero_si128())),
1198 gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i
*)(jjnr
+jidx
)),_mm_setzero_si128())));
1200 jnrA
= (jnrlistA
>=0) ? jnrlistA
: 0;
1201 jnrB
= (jnrlistB
>=0) ? jnrlistB
: 0;
1202 jnrC
= (jnrlistC
>=0) ? jnrlistC
: 0;
1203 jnrD
= (jnrlistD
>=0) ? jnrlistD
: 0;
1204 jnrE
= (jnrlistE
>=0) ? jnrlistE
: 0;
1205 jnrF
= (jnrlistF
>=0) ? jnrlistF
: 0;
1206 jnrG
= (jnrlistG
>=0) ? jnrlistG
: 0;
1207 jnrH
= (jnrlistH
>=0) ? jnrlistH
: 0;
1208 j_coord_offsetA
= DIM
*jnrA
;
1209 j_coord_offsetB
= DIM
*jnrB
;
1210 j_coord_offsetC
= DIM
*jnrC
;
1211 j_coord_offsetD
= DIM
*jnrD
;
1212 j_coord_offsetE
= DIM
*jnrE
;
1213 j_coord_offsetF
= DIM
*jnrF
;
1214 j_coord_offsetG
= DIM
*jnrG
;
1215 j_coord_offsetH
= DIM
*jnrH
;
1217 /* load j atom coordinates */
1218 gmx_mm256_load_1rvec_8ptr_swizzle_ps(x
+j_coord_offsetA
,x
+j_coord_offsetB
,
1219 x
+j_coord_offsetC
,x
+j_coord_offsetD
,
1220 x
+j_coord_offsetE
,x
+j_coord_offsetF
,
1221 x
+j_coord_offsetG
,x
+j_coord_offsetH
,
1224 /* Calculate displacement vector */
1225 dx00
= _mm256_sub_ps(ix0
,jx0
);
1226 dy00
= _mm256_sub_ps(iy0
,jy0
);
1227 dz00
= _mm256_sub_ps(iz0
,jz0
);
1228 dx10
= _mm256_sub_ps(ix1
,jx0
);
1229 dy10
= _mm256_sub_ps(iy1
,jy0
);
1230 dz10
= _mm256_sub_ps(iz1
,jz0
);
1231 dx20
= _mm256_sub_ps(ix2
,jx0
);
1232 dy20
= _mm256_sub_ps(iy2
,jy0
);
1233 dz20
= _mm256_sub_ps(iz2
,jz0
);
1235 /* Calculate squared distance and things based on it */
1236 rsq00
= gmx_mm256_calc_rsq_ps(dx00
,dy00
,dz00
);
1237 rsq10
= gmx_mm256_calc_rsq_ps(dx10
,dy10
,dz10
);
1238 rsq20
= gmx_mm256_calc_rsq_ps(dx20
,dy20
,dz20
);
1240 rinv00
= gmx_mm256_invsqrt_ps(rsq00
);
1241 rinv10
= gmx_mm256_invsqrt_ps(rsq10
);
1242 rinv20
= gmx_mm256_invsqrt_ps(rsq20
);
1244 rinvsq00
= _mm256_mul_ps(rinv00
,rinv00
);
1245 rinvsq10
= _mm256_mul_ps(rinv10
,rinv10
);
1246 rinvsq20
= _mm256_mul_ps(rinv20
,rinv20
);
1248 /* Load parameters for j particles */
1249 jq0
= gmx_mm256_load_8real_swizzle_ps(charge
+jnrA
+0,charge
+jnrB
+0,
1250 charge
+jnrC
+0,charge
+jnrD
+0,
1251 charge
+jnrE
+0,charge
+jnrF
+0,
1252 charge
+jnrG
+0,charge
+jnrH
+0);
1254 fjx0
= _mm256_setzero_ps();
1255 fjy0
= _mm256_setzero_ps();
1256 fjz0
= _mm256_setzero_ps();
1258 /**************************
1259 * CALCULATE INTERACTIONS *
1260 **************************/
1262 if (gmx_mm256_any_lt(rsq00
,rcutoff2
))
1265 r00
= _mm256_mul_ps(rsq00
,rinv00
);
1266 r00
= _mm256_andnot_ps(dummy_mask
,r00
);
1268 /* Compute parameters for interactions between i and j atoms */
1269 qq00
= _mm256_mul_ps(iq0
,jq0
);
1271 /* EWALD ELECTROSTATICS */
1273 /* Analytical PME correction */
1274 zeta2
= _mm256_mul_ps(beta2
,rsq00
);
1275 rinv3
= _mm256_mul_ps(rinvsq00
,rinv00
);
1276 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
1277 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
1278 felec
= _mm256_mul_ps(qq00
,felec
);
1279 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
1280 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
1281 velec
= _mm256_sub_ps(rinv00
,pmecorrV
);
1282 velec
= _mm256_mul_ps(qq00
,velec
);
1284 d
= _mm256_sub_ps(r00
,rswitch
);
1285 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
1286 d2
= _mm256_mul_ps(d
,d
);
1287 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
1289 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
1291 /* Evaluate switch function */
1292 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1293 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv00
,_mm256_mul_ps(velec
,dsw
)) );
1294 cutoff_mask
= _mm256_cmp_ps(rsq00
,rcutoff2
,_CMP_LT_OQ
);
1298 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1300 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1302 /* Calculate temporary vectorial force */
1303 tx
= _mm256_mul_ps(fscal
,dx00
);
1304 ty
= _mm256_mul_ps(fscal
,dy00
);
1305 tz
= _mm256_mul_ps(fscal
,dz00
);
1307 /* Update vectorial force */
1308 fix0
= _mm256_add_ps(fix0
,tx
);
1309 fiy0
= _mm256_add_ps(fiy0
,ty
);
1310 fiz0
= _mm256_add_ps(fiz0
,tz
);
1312 fjx0
= _mm256_add_ps(fjx0
,tx
);
1313 fjy0
= _mm256_add_ps(fjy0
,ty
);
1314 fjz0
= _mm256_add_ps(fjz0
,tz
);
1318 /**************************
1319 * CALCULATE INTERACTIONS *
1320 **************************/
1322 if (gmx_mm256_any_lt(rsq10
,rcutoff2
))
1325 r10
= _mm256_mul_ps(rsq10
,rinv10
);
1326 r10
= _mm256_andnot_ps(dummy_mask
,r10
);
1328 /* Compute parameters for interactions between i and j atoms */
1329 qq10
= _mm256_mul_ps(iq1
,jq0
);
1331 /* EWALD ELECTROSTATICS */
1333 /* Analytical PME correction */
1334 zeta2
= _mm256_mul_ps(beta2
,rsq10
);
1335 rinv3
= _mm256_mul_ps(rinvsq10
,rinv10
);
1336 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
1337 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
1338 felec
= _mm256_mul_ps(qq10
,felec
);
1339 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
1340 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
1341 velec
= _mm256_sub_ps(rinv10
,pmecorrV
);
1342 velec
= _mm256_mul_ps(qq10
,velec
);
1344 d
= _mm256_sub_ps(r10
,rswitch
);
1345 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
1346 d2
= _mm256_mul_ps(d
,d
);
1347 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
1349 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
1351 /* Evaluate switch function */
1352 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1353 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv10
,_mm256_mul_ps(velec
,dsw
)) );
1354 cutoff_mask
= _mm256_cmp_ps(rsq10
,rcutoff2
,_CMP_LT_OQ
);
1358 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1360 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1362 /* Calculate temporary vectorial force */
1363 tx
= _mm256_mul_ps(fscal
,dx10
);
1364 ty
= _mm256_mul_ps(fscal
,dy10
);
1365 tz
= _mm256_mul_ps(fscal
,dz10
);
1367 /* Update vectorial force */
1368 fix1
= _mm256_add_ps(fix1
,tx
);
1369 fiy1
= _mm256_add_ps(fiy1
,ty
);
1370 fiz1
= _mm256_add_ps(fiz1
,tz
);
1372 fjx0
= _mm256_add_ps(fjx0
,tx
);
1373 fjy0
= _mm256_add_ps(fjy0
,ty
);
1374 fjz0
= _mm256_add_ps(fjz0
,tz
);
1378 /**************************
1379 * CALCULATE INTERACTIONS *
1380 **************************/
1382 if (gmx_mm256_any_lt(rsq20
,rcutoff2
))
1385 r20
= _mm256_mul_ps(rsq20
,rinv20
);
1386 r20
= _mm256_andnot_ps(dummy_mask
,r20
);
1388 /* Compute parameters for interactions between i and j atoms */
1389 qq20
= _mm256_mul_ps(iq2
,jq0
);
1391 /* EWALD ELECTROSTATICS */
1393 /* Analytical PME correction */
1394 zeta2
= _mm256_mul_ps(beta2
,rsq20
);
1395 rinv3
= _mm256_mul_ps(rinvsq20
,rinv20
);
1396 pmecorrF
= gmx_mm256_pmecorrF_ps(zeta2
);
1397 felec
= _mm256_add_ps( _mm256_mul_ps(pmecorrF
,beta3
), rinv3
);
1398 felec
= _mm256_mul_ps(qq20
,felec
);
1399 pmecorrV
= gmx_mm256_pmecorrV_ps(zeta2
);
1400 pmecorrV
= _mm256_mul_ps(pmecorrV
,beta
);
1401 velec
= _mm256_sub_ps(rinv20
,pmecorrV
);
1402 velec
= _mm256_mul_ps(qq20
,velec
);
1404 d
= _mm256_sub_ps(r20
,rswitch
);
1405 d
= _mm256_max_ps(d
,_mm256_setzero_ps());
1406 d2
= _mm256_mul_ps(d
,d
);
1407 sw
= _mm256_add_ps(one
,_mm256_mul_ps(d2
,_mm256_mul_ps(d
,_mm256_add_ps(swV3
,_mm256_mul_ps(d
,_mm256_add_ps(swV4
,_mm256_mul_ps(d
,swV5
)))))));
1409 dsw
= _mm256_mul_ps(d2
,_mm256_add_ps(swF2
,_mm256_mul_ps(d
,_mm256_add_ps(swF3
,_mm256_mul_ps(d
,swF4
)))));
1411 /* Evaluate switch function */
1412 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1413 felec
= _mm256_sub_ps( _mm256_mul_ps(felec
,sw
) , _mm256_mul_ps(rinv20
,_mm256_mul_ps(velec
,dsw
)) );
1414 cutoff_mask
= _mm256_cmp_ps(rsq20
,rcutoff2
,_CMP_LT_OQ
);
1418 fscal
= _mm256_and_ps(fscal
,cutoff_mask
);
1420 fscal
= _mm256_andnot_ps(dummy_mask
,fscal
);
1422 /* Calculate temporary vectorial force */
1423 tx
= _mm256_mul_ps(fscal
,dx20
);
1424 ty
= _mm256_mul_ps(fscal
,dy20
);
1425 tz
= _mm256_mul_ps(fscal
,dz20
);
1427 /* Update vectorial force */
1428 fix2
= _mm256_add_ps(fix2
,tx
);
1429 fiy2
= _mm256_add_ps(fiy2
,ty
);
1430 fiz2
= _mm256_add_ps(fiz2
,tz
);
1432 fjx0
= _mm256_add_ps(fjx0
,tx
);
1433 fjy0
= _mm256_add_ps(fjy0
,ty
);
1434 fjz0
= _mm256_add_ps(fjz0
,tz
);
1438 fjptrA
= (jnrlistA
>=0) ? f
+j_coord_offsetA
: scratch
;
1439 fjptrB
= (jnrlistB
>=0) ? f
+j_coord_offsetB
: scratch
;
1440 fjptrC
= (jnrlistC
>=0) ? f
+j_coord_offsetC
: scratch
;
1441 fjptrD
= (jnrlistD
>=0) ? f
+j_coord_offsetD
: scratch
;
1442 fjptrE
= (jnrlistE
>=0) ? f
+j_coord_offsetE
: scratch
;
1443 fjptrF
= (jnrlistF
>=0) ? f
+j_coord_offsetF
: scratch
;
1444 fjptrG
= (jnrlistG
>=0) ? f
+j_coord_offsetG
: scratch
;
1445 fjptrH
= (jnrlistH
>=0) ? f
+j_coord_offsetH
: scratch
;
1447 gmx_mm256_decrement_1rvec_8ptr_swizzle_ps(fjptrA
,fjptrB
,fjptrC
,fjptrD
,fjptrE
,fjptrF
,fjptrG
,fjptrH
,fjx0
,fjy0
,fjz0
);
1449 /* Inner loop uses 321 flops */
1452 /* End of innermost loop */
1454 gmx_mm256_update_iforce_3atom_swizzle_ps(fix0
,fiy0
,fiz0
,fix1
,fiy1
,fiz1
,fix2
,fiy2
,fiz2
,
1455 f
+i_coord_offset
,fshift
+i_shift_offset
);
1457 /* Increment number of inner iterations */
1458 inneriter
+= j_index_end
- j_index_start
;
1460 /* Outer loop uses 18 flops */
1463 /* Increment number of outer iterations */
1466 /* Update outer/inner flops */
1468 inc_nrnb(nrnb
,eNR_NBKERNEL_ELEC_W3_F
,outeriter
*18 + inneriter
*321);