src/gromacs/mdlib/nbnxn_search_simd_2xnn.h

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35
  36 using namespace gmx; // TODO: Remove when this file is moved into gmx namespace
  37
  38 #if GMX_SIMD_REAL_WIDTH >= 2*NBNXN_CPU_CLUSTER_I_SIZE
  39 #define STRIDE_S  (GMX_SIMD_REAL_WIDTH/2)
  40 #else
  41 #define STRIDE_S  NBNXN_CPU_CLUSTER_I_SIZE
  42 #endif
  43
  44 /* Copies PBC shifted i-cell packed atom coordinates to working array */
  45 static inline void
  46 icell_set_x_simd_2xnn(int ci,
  47                       real shx, real shy, real shz,
  48                       int gmx_unused stride, const real *x,
  49                       nbnxn_list_work_t *work)
  50 {
  51     int   ia;
  52     real *x_ci_simd = work->x_ci_simd;
  53
  54     ia = xIndexFromCi<NbnxnLayout::Simd2xNN>(ci);
  55
  56     store(x_ci_simd + 0*GMX_SIMD_REAL_WIDTH, loadU1DualHsimd(x + ia + 0*STRIDE_S + 0) + SimdReal(shx) );
  57     store(x_ci_simd + 1*GMX_SIMD_REAL_WIDTH, loadU1DualHsimd(x + ia + 1*STRIDE_S + 0) + SimdReal(shy) );
  58     store(x_ci_simd + 2*GMX_SIMD_REAL_WIDTH, loadU1DualHsimd(x + ia + 2*STRIDE_S + 0) + SimdReal(shz) );
  59     store(x_ci_simd + 3*GMX_SIMD_REAL_WIDTH, loadU1DualHsimd(x + ia + 0*STRIDE_S + 2) + SimdReal(shx) );
  60     store(x_ci_simd + 4*GMX_SIMD_REAL_WIDTH, loadU1DualHsimd(x + ia + 1*STRIDE_S + 2) + SimdReal(shy) );
  61     store(x_ci_simd + 5*GMX_SIMD_REAL_WIDTH, loadU1DualHsimd(x + ia + 2*STRIDE_S + 2) + SimdReal(shz) );
  62 }
  63
  64 /* SIMD code for checking and adding cluster-pairs to the list using coordinates in packed format.
  65  *
  66  * Checks bouding box distances and possibly atom pair distances.
  67  * This is an accelerated version of make_cluster_list_simple.
  68  *
  69  * \param[in]     gridj               The j-grid
  70  * \param[in,out] nbl                 The pair-list to store the cluster pairs in
  71  * \param[in]     icluster            The index of the i-cluster
  72  * \param[in]     firstCell           The first cluster in the j-range, using i-cluster size indexing
  73  * \param[in]     lastCell            The last cluster in the j-range, using i-cluster size indexing
  74  * \param[in]     excludeSubDiagonal  Exclude atom pairs with i-index > j-index
  75  * \param[in]     x_j                 Coordinates for the j-atom, in SIMD packed format
  76  * \param[in]     rlist2              The squared list cut-off
  77  * \param[in]     rbb2                The squared cut-off for putting cluster-pairs in the list based on bounding box distance only
  78  * \param[in,out] numDistanceChecks   The number of distance checks performed
  79  */
  80 static inline void
  81 makeClusterListSimd2xnn(const nbnxn_grid_t *      gridj,
  82                         nbnxn_pairlist_t *        nbl,
  83                         int                       icluster,
  84                         int                       firstCell,
  85                         int                       lastCell,
  86                         bool                      excludeSubDiagonal,
  87                         const real * gmx_restrict x_j,
  88                         real                      rlist2,
  89                         float                     rbb2,
  90                         int * gmx_restrict        numDistanceChecks)
  91 {
  92     const real * gmx_restrict           x_ci_simd = nbl->work->x_ci_simd;
  93     const nbnxn_bb_t * gmx_restrict     bb_ci     = nbl->work->bb_ci;
  94
  95     SimdReal                            jx_S, jy_S, jz_S;
  96
  97     SimdReal                            dx_S0, dy_S0, dz_S0;
  98     SimdReal                            dx_S2, dy_S2, dz_S2;
  99
 100     SimdReal                            rsq_S0;
 101     SimdReal                            rsq_S2;
 102
 103     SimdBool                            wco_S0;
 104     SimdBool                            wco_S2;
 105     SimdBool                            wco_any_S;
 106
 107     SimdReal                            rc2_S;
 108
 109     gmx_bool                            InRange;
 110     float                               d2;
 111     int                                 xind_f, xind_l;
 112
 113     int jclusterFirst = cjFromCi<NbnxnLayout::Simd2xNN>(firstCell);
 114     int jclusterLast  = cjFromCi<NbnxnLayout::Simd2xNN>(lastCell);
 115     GMX_ASSERT(jclusterLast >= jclusterFirst, "We should have a non-empty j-cluster range, since the calling code should have ensured a non-empty cell range");
 116
 117     rc2_S   = SimdReal(rlist2);
 118
 119     InRange = FALSE;
 120     while (!InRange && jclusterFirst <= jclusterLast)
 121     {
 122 #if NBNXN_SEARCH_BB_SIMD4
 123         d2 = subc_bb_dist2_simd4(0, bb_ci, jclusterFirst, gridj->bbj);
 124 #else
 125         d2 = subc_bb_dist2(0, bb_ci, jclusterFirst, gridj->bbj);
 126 #endif
 127         *numDistanceChecks += 2;
 128
 129         /* Check if the distance is within the distance where
 130          * we use only the bounding box distance rbb,
 131          * or within the cut-off and there is at least one atom pair
 132          * within the cut-off.
 133          */
 134         if (d2 < rbb2)
 135         {
 136             InRange = TRUE;
 137         }
 138         else if (d2 < rlist2)
 139         {
 140             xind_f  = xIndexFromCj<NbnxnLayout::Simd2xNN>(cjFromCi<NbnxnLayout::Simd2xNN>(gridj->cell0) + jclusterFirst);
 141
 142             jx_S  = loadDuplicateHsimd(x_j + xind_f + 0*STRIDE_S);
 143             jy_S  = loadDuplicateHsimd(x_j + xind_f + 1*STRIDE_S);
 144             jz_S  = loadDuplicateHsimd(x_j + xind_f + 2*STRIDE_S);
 145
 146             /* Calculate distance */
 147             dx_S0            = load<SimdReal>(x_ci_simd + 0*GMX_SIMD_REAL_WIDTH) - jx_S;
 148             dy_S0            = load<SimdReal>(x_ci_simd + 1*GMX_SIMD_REAL_WIDTH) - jy_S;
 149             dz_S0            = load<SimdReal>(x_ci_simd + 2*GMX_SIMD_REAL_WIDTH) - jz_S;
 150             dx_S2            = load<SimdReal>(x_ci_simd + 3*GMX_SIMD_REAL_WIDTH) - jx_S;
 151             dy_S2            = load<SimdReal>(x_ci_simd + 4*GMX_SIMD_REAL_WIDTH) - jy_S;
 152             dz_S2            = load<SimdReal>(x_ci_simd + 5*GMX_SIMD_REAL_WIDTH) - jz_S;
 153
 154             /* rsq = dx*dx+dy*dy+dz*dz */
 155             rsq_S0           = norm2(dx_S0, dy_S0, dz_S0);
 156             rsq_S2           = norm2(dx_S2, dy_S2, dz_S2);
 157
 158             wco_S0           = (rsq_S0 < rc2_S);
 159             wco_S2           = (rsq_S2 < rc2_S);
 160
 161             wco_any_S        = wco_S0 || wco_S2;
 162
 163             InRange          = anyTrue(wco_any_S);
 164
 165             *numDistanceChecks += 2*GMX_SIMD_REAL_WIDTH;
 166         }
 167         if (!InRange)
 168         {
 169             jclusterFirst++;
 170         }
 171     }
 172     if (!InRange)
 173     {
 174         return;
 175     }
 176
 177     InRange = FALSE;
 178     while (!InRange && jclusterLast > jclusterFirst)
 179     {
 180 #if NBNXN_SEARCH_BB_SIMD4
 181         d2 = subc_bb_dist2_simd4(0, bb_ci, jclusterLast, gridj->bbj);
 182 #else
 183         d2 = subc_bb_dist2(0, bb_ci, jclusterLast, gridj->bbj);
 184 #endif
 185         *numDistanceChecks += 2;
 186
 187         /* Check if the distance is within the distance where
 188          * we use only the bounding box distance rbb,
 189          * or within the cut-off and there is at least one atom pair
 190          * within the cut-off.
 191          */
 192         if (d2 < rbb2)
 193         {
 194             InRange = TRUE;
 195         }
 196         else if (d2 < rlist2)
 197         {
 198             xind_l  = xIndexFromCj<NbnxnLayout::Simd2xNN>(cjFromCi<NbnxnLayout::Simd2xNN>(gridj->cell0) + jclusterLast);
 199
 200             jx_S  = loadDuplicateHsimd(x_j + xind_l + 0*STRIDE_S);
 201             jy_S  = loadDuplicateHsimd(x_j + xind_l + 1*STRIDE_S);
 202             jz_S  = loadDuplicateHsimd(x_j + xind_l + 2*STRIDE_S);
 203
 204             /* Calculate distance */
 205             dx_S0            = load<SimdReal>(x_ci_simd + 0*GMX_SIMD_REAL_WIDTH) - jx_S;
 206             dy_S0            = load<SimdReal>(x_ci_simd + 1*GMX_SIMD_REAL_WIDTH) - jy_S;
 207             dz_S0            = load<SimdReal>(x_ci_simd + 2*GMX_SIMD_REAL_WIDTH) - jz_S;
 208             dx_S2            = load<SimdReal>(x_ci_simd + 3*GMX_SIMD_REAL_WIDTH) - jx_S;
 209             dy_S2            = load<SimdReal>(x_ci_simd + 4*GMX_SIMD_REAL_WIDTH) - jy_S;
 210             dz_S2            = load<SimdReal>(x_ci_simd + 5*GMX_SIMD_REAL_WIDTH) - jz_S;
 211
 212             /* rsq = dx*dx+dy*dy+dz*dz */
 213             rsq_S0           = norm2(dx_S0, dy_S0, dz_S0);
 214             rsq_S2           = norm2(dx_S2, dy_S2, dz_S2);
 215
 216             wco_S0           = (rsq_S0 < rc2_S);
 217             wco_S2           = (rsq_S2 < rc2_S);
 218
 219             wco_any_S        = wco_S0 || wco_S2;
 220
 221             InRange          = anyTrue(wco_any_S);
 222
 223             *numDistanceChecks += 2*GMX_SIMD_REAL_WIDTH;
 224         }
 225         if (!InRange)
 226         {
 227             jclusterLast--;
 228         }
 229     }
 230
 231     if (jclusterFirst <= jclusterLast)
 232     {
 233         for (int jcluster = jclusterFirst; jcluster <= jclusterLast; jcluster++)
 234         {
 235             /* Store cj and the interaction mask */
 236             nbl->cj[nbl->ncj].cj   = cjFromCi<NbnxnLayout::Simd2xNN>(gridj->cell0) + jcluster;
 237             nbl->cj[nbl->ncj].excl = get_imask_simd_2xnn(excludeSubDiagonal, icluster, jcluster);
 238             nbl->ncj++;
 239         }
 240         /* Increase the closing index in i super-cell list */
 241         nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
 242     }
 243 }
 244
 245 #undef STRIDE_S