2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014,2015,2017,2018,2019, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 // FIXME: remove the "__" prefix in front of the group def when we move the
37 // nonbonded code into separate dir.
39 /*! \libinternal \defgroup __module_nbnxm Short-range non-bonded interaction module
40 * \ingroup group_mdrun
42 * \brief Computes forces and energies for short-range pair-interactions
43 * based on the Verlet algorithm. The algorithm uses pair-lists generated
44 * at fixed intervals as well as various flavors of pair interaction kernels
45 * implemented for a wide range of CPU and GPU architectures.
47 * The module includes support for flavors of Coulomb and Lennard-Jones interaction
48 * treatment implemented for a large range of SIMD instruction sets for CPU
49 * architectures as well as in CUDA and OpenCL for GPU architectures.
50 * Additionally there is a reference CPU non-SIMD and a reference CPU
51 * for GPU pair-list setup interaction kernel.
53 * The implementation of the kernels is based on the cluster non-bonded algorithm
54 * which in the code is referred to as the NxM algorithms ("nbnxm_" prefix);
55 * for details of the algorithm see DOI:10.1016/j.cpc.2013.06.003.
57 * Algorithmically, the non-bonded computation has two different modes:
58 * A "classical" mode: generate a list every nstlist steps containing at least
59 * all atom pairs up to a distance of rlistOuter and compute pair interactions
60 * for all pairs that are within the interaction cut-off.
61 * A "dynamic pruning" mode: generate an "outer-list" up to cut-off rlistOuter
62 * every nstlist steps and prune the outer-list using a cut-off of rlistInner
63 * every nstlistPrune steps to obtain a, smaller, "inner-list". This
64 * results in fewer interaction computations and allows for a larger nstlist.
65 * On a GPU, this dynamic pruning is performed in a rolling fashion, pruning
66 * only a sub-part of the list each (second) step. This way it can often
67 * overlap with integration and constraints on the CPU.
68 * Currently a simple heuristic determines which mode will be used.
70 * TODO: add a summary list and brief descriptions of the different submodules:
71 * search, CPU kernels, GPU glue code + kernels.
73 * \author Berk Hess <hess@kth.se>
74 * \author Szilárd Páll <pall.szilard@gmail.com>
75 * \author Mark Abraham <mark.j.abraham@gmail.com>
76 * \author Anca Hamuraru <anca@streamcomputing.eu>
77 * \author Teemu Virolainen <teemu@streamcomputing.eu>
78 * \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
80 * TODO: add more authors!
84 * \defgroup module_nbnxm Non-bonded pair interactions
85 * \ingroup group_mdrun
87 * Implements non-bonded pair interaction functionality for NxM atom clusters.
89 * This module provides methods to, very efficiently, compute non-bonded
90 * pair interactions on CPUs as well as accelerators. It also provides
91 * a method to construct the NxM atom-cluster pair-list required for
92 * computing these non-bonded iteractions.
95 /*! \libinternal \file
97 * \brief This file contains the public interface of the nbnxm module
98 * that implements the NxM atom cluster non-bonded algorithm to efficiently
99 * compute pair forces.
102 * \author Berk Hess <hess@kth.se>
103 * \author Szilárd Páll <pall.szilard@gmail.com>
106 * \ingroup module_nbnxm
110 #ifndef GMX_NBNXM_NBNXM_H
111 #define GMX_NBNXM_NBNXM_H
115 #include "gromacs/gpu_utils/devicebuffer_datatype.h"
116 #include "gromacs/math/vectypes.h"
117 #include "gromacs/utility/arrayref.h"
118 #include "gromacs/utility/enumerationhelpers.h"
119 #include "gromacs/utility/real.h"
121 #include "locality.h"
123 // TODO: Remove this include and the two nbnxm includes above
124 #include "nbnxm_gpu.h"
126 struct gmx_device_info_t
;
127 struct gmx_domdec_zones_t
;
128 struct gmx_enerdata_t
;
129 struct gmx_hw_info_t
;
131 struct gmx_wallcycle
;
132 struct interaction_const_t
;
133 struct nonbonded_verlet_t
;
144 /*! \brief Switch for whether to use GPU for buffer ops*/
145 enum class BufferOpsUseGpu
151 class GpuEventSynchronizer
;
155 class ForceWithShiftForces
;
157 class UpdateGroupsCog
;
162 enum class KernelType
;
168 /*! \brief Nonbonded NxN kernel types: plain C, CPU SIMD, GPU, GPU emulation */
169 enum class KernelType
: int
180 /*! \brief Ewald exclusion types */
181 enum class EwaldExclusionType
: int
189 /* \brief The non-bonded setup, also affects the pairlist construction kernel */
192 //! The non-bonded type, also affects the pairlist construction kernel
193 KernelType kernelType
= KernelType::NotSet
;
194 //! Ewald exclusion computation handling type, currently only used for CPU
195 EwaldExclusionType ewaldExclusionType
= EwaldExclusionType::NotSet
;
198 /*! \brief Return a string identifying the kernel type.
200 * \param [in] kernelType nonbonded kernel type, takes values from the nbnxn_kernel_type enum
201 * \returns a string identifying the kernel corresponding to the type passed as argument
203 const char *lookup_kernel_name(Nbnxm::KernelType kernelType
);
207 /*! \brief Flag to tell the nonbonded kernels whether to clear the force output buffers */
209 enbvClearFNo
, enbvClearFYes
213 * \brief Top-level non-bonded data structure for the Verlet-type cut-off scheme. */
214 struct nonbonded_verlet_t
217 //! Constructs an object from its components
218 nonbonded_verlet_t(std::unique_ptr
<PairlistSets
> pairlistSets
,
219 std::unique_ptr
<PairSearch
> pairSearch
,
220 std::unique_ptr
<nbnxn_atomdata_t
> nbat
,
221 const Nbnxm::KernelSetup
&kernelSetup
,
222 gmx_nbnxn_gpu_t
*gpu_nbv
,
223 gmx_wallcycle
*wcycle
);
225 ~nonbonded_verlet_t();
227 //! Returns whether a GPU is use for the non-bonded calculations
230 return kernelSetup_
.kernelType
== Nbnxm::KernelType::Gpu8x8x8
;
233 //! Returns whether a GPU is emulated for the non-bonded calculations
234 bool emulateGpu() const
236 return kernelSetup_
.kernelType
== Nbnxm::KernelType::Cpu8x8x8_PlainC
;
239 //! Return whether the pairlist is of simple, CPU type
240 bool pairlistIsSimple() const
242 return !useGpu() && !emulateGpu();
245 //! Initialize the pair list sets, TODO this should be private
246 void initPairlistSets(bool haveMultipleDomains
);
248 //! Returns the order of the local atoms on the grid
249 gmx::ArrayRef
<const int> getLocalAtomOrder() const;
251 //! Sets the order of the local atoms to the order grid atom ordering
252 void setLocalAtomOrder();
254 //! Returns the index position of the atoms on the search grid
255 gmx::ArrayRef
<const int> getGridIndices() const;
257 //! Constructs the pairlist for the given locality
258 void constructPairlist(Nbnxm::InteractionLocality iLocality
,
259 const t_blocka
*excl
,
263 //! Updates all the atom properties in Nbnxm
264 void setAtomProperties(const t_mdatoms
&mdatoms
,
265 gmx::ArrayRef
<const int> atomInfo
);
267 /*!\brief Convert the coordinates to NBNXM format for the given locality.
269 * The API function for the transformation of the coordinates from one layout to another.
271 * \param[in] locality Whether coordinates for local or non-local atoms should be transformed.
272 * \param[in] fillLocal If the coordinates for filler particles should be zeroed.
273 * \param[in] coordinates Coordinates in plain rvec format to be transformed.
275 void convertCoordinates(Nbnxm::AtomLocality locality
,
277 gmx::ArrayRef
<const gmx::RVec
> coordinates
);
279 /*!\brief Convert the coordinates to NBNXM format on the GPU for the given locality
281 * The API function for the transformation of the coordinates from one layout to another in the GPU memory.
283 * \param[in] locality Whether coordinates for local or non-local atoms should be transformed.
284 * \param[in] fillLocal If the coordinates for filler particles should be zeroed.
285 * \param[in] d_x GPU coordinates buffer in plain rvec format to be transformed.
286 * \param[in] xReadyOnDevice Event synchronizer indicating that the coordinates are ready in the device memory.
288 void convertCoordinatesGpu(Nbnxm::AtomLocality locality
,
290 DeviceBuffer
<float> d_x
,
291 GpuEventSynchronizer
*xReadyOnDevice
);
293 //! Init for GPU version of setup coordinates in Nbnxm
294 void atomdata_init_copy_x_to_nbat_x_gpu();
296 //! Sync the nonlocal GPU stream with dependent tasks in the local queue.
297 void insertNonlocalGpuDependency(Nbnxm::InteractionLocality interactionLocality
);
299 //! Returns a reference to the pairlist sets
300 const PairlistSets
&pairlistSets() const
302 return *pairlistSets_
;
305 //! Returns whether step is a dynamic list pruning step, for CPU lists
306 bool isDynamicPruningStepCpu(int64_t step
) const;
308 //! Returns whether step is a dynamic list pruning step, for GPU lists
309 bool isDynamicPruningStepGpu(int64_t step
) const;
311 //! Dispatches the dynamic pruning kernel for the given locality, for CPU lists
312 void dispatchPruneKernelCpu(Nbnxm::InteractionLocality iLocality
,
313 const rvec
*shift_vec
);
315 //! Dispatches the dynamic pruning kernel for GPU lists
316 void dispatchPruneKernelGpu(int64_t step
);
318 //! \brief Executes the non-bonded kernel of the GPU or launches it on the GPU
319 void dispatchNonbondedKernel(Nbnxm::InteractionLocality iLocality
,
320 const interaction_const_t
&ic
,
321 const gmx::StepWorkload
&stepWork
,
323 const t_forcerec
&fr
,
324 gmx_enerdata_t
*enerd
,
327 //! Executes the non-bonded free-energy kernel, always runs on the CPU
328 void dispatchFreeEnergyKernel(Nbnxm::InteractionLocality iLocality
,
329 const t_forcerec
*fr
,
331 gmx::ForceWithShiftForces
*forceWithShiftForces
,
332 const t_mdatoms
&mdatoms
,
335 gmx_enerdata_t
*enerd
,
336 const gmx::StepWorkload
&stepWork
,
339 /*! \brief Add the forces stored in nbat to f, zeros the forces in nbat
340 * \param [in] locality Local or non-local
341 * \param [inout] force Force to be added to
343 void atomdata_add_nbat_f_to_f(Nbnxm::AtomLocality locality
,
344 gmx::ArrayRef
<gmx::RVec
> force
);
346 /*! \brief Add the forces stored in nbat to total force using GPU buffer opse
348 * \param [in] locality Local or non-local
349 * \param [in,out] totalForcesDevice Force to be added to
350 * \param [in] forcesPmeDevice Device buffer with PME forces
351 * \param[in] dependencyList List of synchronizers that represent the dependencies the reduction task needs to sync on.
352 * \param [in] useGpuFPmeReduction Whether PME forces should be added
353 * \param [in] accumulateForce If the total force buffer already contains data
355 void atomdata_add_nbat_f_to_f_gpu(Nbnxm::AtomLocality locality
,
356 DeviceBuffer
<float> totalForcesDevice
,
357 void *forcesPmeDevice
,
358 gmx::ArrayRef
<GpuEventSynchronizer
* const> dependencyList
,
359 bool useGpuFPmeReduction
,
360 bool accumulateForce
);
362 /*! \brief Outer body of function to perform initialization for F buffer operations on GPU. */
363 void atomdata_init_add_nbat_f_to_f_gpu();
365 /*! \brief return pointer to GPU event recorded when coordinates have been copied to device */
366 void* get_x_on_device_event();
368 /*! \brief Wait for non-local copy of coordinate buffer from device to host */
369 void wait_nonlocal_x_copy_D2H_done();
371 /*! \brief return GPU pointer to f in rvec format */
372 void* get_gpu_frvec();
374 /*! \brief Ensure local stream waits for non-local stream */
375 void stream_local_wait_for_nonlocal();
377 //! Return the kernel setup
378 const Nbnxm::KernelSetup
&kernelSetup() const
383 //! Returns the outer radius for the pair list
384 real
pairlistInnerRadius() const;
386 //! Returns the outer radius for the pair list
387 real
pairlistOuterRadius() const;
389 //! Changes the pair-list outer and inner radius
390 void changePairlistRadii(real rlistOuter
,
393 //! Set up internal flags that indicate what type of short-range work there is.
394 void setupGpuShortRangeWork(const gmx::GpuBonded
*gpuBonded
,
395 const Nbnxm::InteractionLocality iLocality
)
397 if (useGpu() && !emulateGpu())
399 Nbnxm::setupGpuShortRangeWork(gpu_nbv
, gpuBonded
, iLocality
);
403 //! Returns true if there is GPU short-range work for the given atom locality.
404 bool haveGpuShortRangeWork(const Nbnxm::AtomLocality aLocality
)
406 return ((useGpu() && !emulateGpu()) &&
407 Nbnxm::haveGpuShortRangeWork(gpu_nbv
, aLocality
));
410 // TODO: Make all data members private
412 //! All data related to the pair lists
413 std::unique_ptr
<PairlistSets
> pairlistSets_
;
414 //! Working data for constructing the pairlists
415 std::unique_ptr
<PairSearch
> pairSearch_
;
417 std::unique_ptr
<nbnxn_atomdata_t
> nbat
;
419 //! The non-bonded setup, also affects the pairlist construction kernel
420 Nbnxm::KernelSetup kernelSetup_
;
421 //! \brief Pointer to wallcycle structure.
422 gmx_wallcycle
*wcycle_
;
424 //! GPU Nbnxm data, only used with a physical GPU (TODO: use unique_ptr)
425 gmx_nbnxn_gpu_t
*gpu_nbv
;
431 /*! \brief Creates an Nbnxm object */
432 std::unique_ptr
<nonbonded_verlet_t
>
433 init_nb_verlet(const gmx::MDLogger
&mdlog
,
434 gmx_bool bFEP_NonBonded
,
435 const t_inputrec
*ir
,
436 const t_forcerec
*fr
,
438 const gmx_hw_info_t
&hardwareInfo
,
439 const gmx_device_info_t
*deviceInfo
,
440 const gmx_mtop_t
*mtop
,
442 gmx_wallcycle
*wcycle
);
446 /*! \brief Put the atoms on the pair search grid.
448 * Only atoms atomStart to atomEnd in x are put on the grid.
449 * The atom_density is used to determine the grid size.
450 * When atomDensity<=0, the density is determined from atomEnd-atomStart and the corners.
451 * With domain decomposition part of the n particles might have migrated,
452 * but have not been removed yet. This count is given by nmoved.
453 * When move[i] < 0 particle i has migrated and will not be put on the grid.
454 * Without domain decomposition move will be NULL.
456 void nbnxn_put_on_grid(nonbonded_verlet_t
*nb_verlet
,
459 const rvec lowerCorner
,
460 const rvec upperCorner
,
461 const gmx::UpdateGroupsCog
*updateGroupsCog
,
465 gmx::ArrayRef
<const int> atomInfo
,
466 gmx::ArrayRef
<const gmx::RVec
> x
,
470 /*! \brief As nbnxn_put_on_grid, but for the non-local atoms
472 * with domain decomposition. Should be called after calling
473 * nbnxn_search_put_on_grid for the local atoms / home zone.
475 void nbnxn_put_on_grid_nonlocal(nonbonded_verlet_t
*nb_verlet
,
476 const struct gmx_domdec_zones_t
*zones
,
477 gmx::ArrayRef
<const int> atomInfo
,
478 gmx::ArrayRef
<const gmx::RVec
> x
);
480 #endif // GMX_NBNXN_NBNXM_H