2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 /*! \libinternal \file
36 * \brief Declares functionality for deciding whether tasks will run on GPUs.
38 * \author Mark Abraham <mark.j.abraham@gmail.com>
39 * \ingroup module_taskassignment
43 #ifndef GMX_TASKASSIGNMENT_DECIDEGPUUSAGE_H
44 #define GMX_TASKASSIGNMENT_DECIDEGPUUSAGE_H
51 enum class PmeRunMode
;
58 //! Record where a compute task is targetted.
59 enum class TaskTarget
: int
66 //! Help pass GPU-emulation parameters with type safety.
67 enum class EmulateGpuNonbonded
: bool
69 //! Do not emulate GPUs.
76 * \brief Structure that holds boolean flags corresponding to the development
77 * features present enabled through environment variables.
80 struct DevelopmentFeatureFlags
82 //! True if the Buffer ops development feature is enabled
83 // TODO: when the trigger of the buffer ops offload is fully automated this should go away
84 bool enableGpuBufferOps
= false;
85 //! If true, forces 'mdrun -update auto' default to 'gpu'
86 bool forceGpuUpdateDefault
= false;
87 //! True if the GPU halo exchange development feature is enabled
88 bool enableGpuHaloExchange
= false;
89 //! True if the PME PP direct communication GPU development feature is enabled
90 bool enableGpuPmePPComm
= false;
96 /*! \brief Decide whether this thread-MPI simulation will run
97 * nonbonded tasks on GPUs.
99 * The number of GPU tasks and devices influences both the choice of
100 * the number of ranks, and checks upon any such choice made by the
101 * user. So we need to consider this before any automated choice of
102 * the number of thread-MPI ranks.
104 * \param[in] nonbondedTarget The user's choice for mdrun -nb for where to assign short-ranged nonbonded interaction tasks.
105 * \param[in] gpuIdsToUse The compatible GPUs that the user permitted us to use.
106 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
107 * \param[in] emulateGpuNonbonded Whether we will emulate GPU calculation of nonbonded interactions.
108 * \param[in] buildSupportsNonbondedOnGpu Whether GROMACS was built with GPU support.
109 * \param[in] nonbondedOnGpuIsUseful Whether computing nonbonded interactions on a GPU is useful for this calculation.
110 * \param[in] numRanksPerSimulation The number of ranks in each simulation.
112 * \returns Whether the simulation will run nonbonded tasks on GPUs.
114 * \throws std::bad_alloc If out of memory
115 * InconsistentInputError If the user requirements are inconsistent. */
116 bool decideWhetherToUseGpusForNonbondedWithThreadMpi(TaskTarget nonbondedTarget
,
117 const std::vector
<int>& gpuIdsToUse
,
118 const std::vector
<int>& userGpuTaskAssignment
,
119 EmulateGpuNonbonded emulateGpuNonbonded
,
120 bool buildSupportsNonbondedOnGpu
,
121 bool nonbondedOnGpuIsUseful
,
122 int numRanksPerSimulation
);
124 /*! \brief Decide whether this thread-MPI simulation will run
127 * The number of GPU tasks and devices influences both the choice of
128 * the number of ranks, and checks upon any such choice made by the
129 * user. So we need to consider this before any automated choice of
130 * the number of thread-MPI ranks.
132 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
133 * \param[in] pmeTarget The user's choice for mdrun -pme for where to assign
134 * long-ranged PME nonbonded interaction tasks.
135 * \param[in] gpuIdsToUse The compatible GPUs that the user permitted us to use.
136 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
137 * \param[in] hardwareInfo Hardware information
138 * \param[in] inputrec The user input
139 * \param[in] numRanksPerSimulation The number of ranks in each simulation.
140 * \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation.
142 * \returns Whether the simulation will run PME tasks on GPUs.
144 * \throws std::bad_alloc If out of memory
145 * InconsistentInputError If the user requirements are inconsistent. */
146 bool decideWhetherToUseGpusForPmeWithThreadMpi(bool useGpuForNonbonded
,
147 TaskTarget pmeTarget
,
148 const std::vector
<int>& gpuIdsToUse
,
149 const std::vector
<int>& userGpuTaskAssignment
,
150 const gmx_hw_info_t
& hardwareInfo
,
151 const t_inputrec
& inputrec
,
152 int numRanksPerSimulation
,
153 int numPmeRanksPerSimulation
);
155 /*! \brief Decide whether the simulation will try to run nonbonded
158 * The final decision cannot be made until after the duty of the rank
159 * is known. But we need to know if nonbonded will run on GPUs for
160 * setting up DD (particularly rlist) and determining duty. If the
161 * user requires GPUs for the tasks of that duty, then it will be an
162 * error when none are found.
164 * With thread-MPI, calls have been made to
165 * decideWhetherToUseGpusForNonbondedWithThreadMpi() and
166 * decideWhetherToUseGpusForPmeWithThreadMpi() to help determine
167 * the number of ranks and run some checks, but the final
168 * decision is made in this routine, along with many more
169 * consistency checks.
171 * \param[in] nonbondedTarget The user's choice for mdrun -nb for where to assign short-ranged nonbonded interaction tasks.
172 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
173 * \param[in] emulateGpuNonbonded Whether we will emulate GPU calculation of nonbonded interactions.
174 * \param[in] buildSupportsNonbondedOnGpu Whether GROMACS was build with GPU support.
175 * \param[in] nonbondedOnGpuIsUseful Whether computing nonbonded interactions on a GPU is useful for this calculation.
176 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
178 * \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
180 * \throws std::bad_alloc If out of memory
181 * InconsistentInputError If the user requirements are inconsistent. */
182 bool decideWhetherToUseGpusForNonbonded(TaskTarget nonbondedTarget
,
183 const std::vector
<int>& userGpuTaskAssignment
,
184 EmulateGpuNonbonded emulateGpuNonbonded
,
185 bool buildSupportsNonbondedOnGpu
,
186 bool nonbondedOnGpuIsUseful
,
187 bool gpusWereDetected
);
189 /*! \brief Decide whether the simulation will try to run tasks of
190 * different types on GPUs.
192 * The final decision cannot be made until after the duty of the rank
193 * is known. But we need to know if nonbonded will run on GPUs for
194 * setting up DD (particularly rlist) and determining duty. If the
195 * user requires GPUs for the tasks of that duty, then it will be an
196 * error when none are found.
198 * With thread-MPI, calls have been made to
199 * decideWhetherToUseGpusForNonbondedWithThreadMpi() and
200 * decideWhetherToUseGpusForPmeWithThreadMpi() to help determine
201 * the number of ranks and run some checks, but the final
202 * decision is made in this routine, along with many more
203 * consistency checks.
205 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
206 * \param[in] pmeTarget The user's choice for mdrun -pme for where to assign long-ranged PME nonbonded interaction tasks.
207 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
208 * \param[in] hardwareInfo Hardware information
209 * \param[in] inputrec The user input
210 * \param[in] numRanksPerSimulation The number of ranks in each simulation.
211 * \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation.
212 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
214 * \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
216 * \throws std::bad_alloc If out of memory
217 * InconsistentInputError If the user requirements are inconsistent. */
218 bool decideWhetherToUseGpusForPme(bool useGpuForNonbonded
,
219 TaskTarget pmeTarget
,
220 const std::vector
<int>& userGpuTaskAssignment
,
221 const gmx_hw_info_t
& hardwareInfo
,
222 const t_inputrec
& inputrec
,
223 int numRanksPerSimulation
,
224 int numPmeRanksPerSimulation
,
225 bool gpusWereDetected
);
227 /*! \brief Determine PME run mode.
229 * Given the PME task assignment in \p useGpuForPme and the user-provided
230 * FFT task target in \p pmeFftTarget, returns a PME run mode for the
231 * current run. It also checks the compatibility of the two.
233 * \note Aborts the run upon incompatible values of \p useGpuForPme and \p pmeFftTarget.
235 * \param[in] useGpuForPme PME task assignment, true if PME task is mapped to the GPU.
236 * \param[in] pmeFftTarget The user's choice for -pmefft for where to assign the FFT
237 * work of the PME task. \param[in] inputrec The user input record
239 PmeRunMode
determinePmeRunMode(bool useGpuForPme
, const TaskTarget
& pmeFftTarget
, const t_inputrec
& inputrec
);
241 /*! \brief Decide whether the simulation will try to run bonded tasks on GPUs.
243 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
244 * \param[in] useGpuForPme Whether GPUs will be used for PME interactions.
245 * \param[in] bondedTarget The user's choice for mdrun -bonded for where to assign tasks.
246 * \param[in] canUseGpuForBonded Whether the bonded interactions can run on a GPU
247 * \param[in] usingLJPme Whether Vdw interactions use LJ-PME.
248 * \param[in] usingElecPmeOrEwald Whether a PME or Ewald type method is used for electrostatics.
249 * \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation, can be -1 for auto.
250 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
252 * \returns Whether the simulation will run bondeded tasks on GPUs.
254 * \throws std::bad_alloc If out of memory
255 * InconsistentInputError If the user requirements are inconsistent. */
256 bool decideWhetherToUseGpusForBonded(bool useGpuForNonbonded
,
258 TaskTarget bondedTarget
,
259 bool canUseGpuForBonded
,
261 bool usingElecPmeOrEwald
,
262 int numPmeRanksPerSimulation
,
263 bool gpusWereDetected
);
265 /*! \brief Decide whether to use GPU for update.
267 * \param[in] isDomainDecomposition Whether there more than one domain.
268 * \param[in] useUpdateGroups If the constraints can be split across domains.
269 * \param[in] pmeRunMode PME running mode: CPU, GPU or mixed.
270 * \param[in] havePmeOnlyRank If there is a PME-only rank in the simulation.
271 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
272 * \param[in] updateTarget User choice for running simulation on GPU.
273 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
274 * \param[in] inputrec The user input.
275 * \param[in] mtop The global topology.
276 * \param[in] useEssentialDynamics If essential dynamics is active.
277 * \param[in] doOrientationRestraints If orientation restraints are enabled.
278 * \param[in] useReplicaExchange If this is a REMD simulation.
279 * \param[in] doRerun It this is a rerun.
280 * \param[in] devFlags GPU development / experimental feature flags.
281 * \param[in] mdlog MD logger.
283 * \returns Whether complete simulation can be run on GPU.
284 * \throws std::bad_alloc If out of memory
285 * InconsistentInputError If the user requirements are inconsistent.
287 bool decideWhetherToUseGpuForUpdate(bool isDomainDecomposition
,
288 bool useUpdateGroups
,
289 PmeRunMode pmeRunMode
,
290 bool havePmeOnlyRank
,
291 bool useGpuForNonbonded
,
292 TaskTarget updateTarget
,
293 bool gpusWereDetected
,
294 const t_inputrec
& inputrec
,
295 const gmx_mtop_t
& mtop
,
296 bool useEssentialDynamics
,
297 bool doOrientationRestraints
,
298 bool useReplicaExchange
,
300 const DevelopmentFeatureFlags
& devFlags
,
301 const gmx::MDLogger
& mdlog
);
304 /*! \brief Decide whether to use GPU for halo exchange.
306 * \param[in] devFlags GPU development / experimental feature flags.
307 * \param[in] havePPDomainDecomposition Whether PP domain decomposition is in use.
308 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
309 * \param[in] useModularSimulator Whether modularsimulator is in use.
310 * \param[in] doRerun Whether this is a rerun.
311 * \param[in] haveEnergyMinimization Whether energy minimization is in use.
313 * \returns Whether halo exchange can be run on GPU.
315 bool decideWhetherToUseGpuForHalo(const DevelopmentFeatureFlags
& devFlags
,
316 bool havePPDomainDecomposition
,
317 bool useGpuForNonbonded
,
318 bool useModularSimulator
,
320 bool haveEnergyMinimization
);