2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * \brief Tests GPU stream manager
38 * \author Mark Abraham <mark.j.abraham@gmail.com>
39 * \author Artem Zhmurov <zhmurov@gmail.com>
41 * \ingroup module_gpu_utils
45 #include "gromacs/gpu_utils/device_stream_manager.h"
49 #include <initializer_list>
52 #include <gtest/gtest.h>
54 #include "gromacs/hardware/device_management.h"
55 #include "gromacs/mdtypes/simulation_workload.h"
56 #include "gromacs/utility/enumerationhelpers.h"
58 #include "testutils/test_hardware_environment.h"
69 //! GPU device stream names for outputs.
70 const EnumerationArray
<DeviceStreamType
, std::string
> c_deviceStreamNames
= {
71 { "non-bonded local", "non-bonded non-local", "PME", "PME-PP transfer", "update" }
74 /*! \brief Non-GPU builds return nullptr instead of streams,
75 * so we have to expect that in such build configurations. */
76 constexpr bool c_canExpectValidStreams
= (GMX_GPU
!= 0);
78 //! Helper function to implement readable testing
79 void expectValidStreams(DeviceStreamManager
* manager
, std::initializer_list
<DeviceStreamType
> types
)
81 if (c_canExpectValidStreams
)
83 for (const DeviceStreamType type
: types
)
85 SCOPED_TRACE("Testing " + c_deviceStreamNames
[type
] + " stream.");
86 EXPECT_TRUE(manager
->streamIsValid(type
));
90 //! Helper function to implement readable testing
91 void expectInvalidStreams(DeviceStreamManager
* manager
, std::initializer_list
<DeviceStreamType
> types
)
93 for (const DeviceStreamType type
: types
)
95 SCOPED_TRACE("Testing " + c_deviceStreamNames
[type
] + " stream.");
96 EXPECT_FALSE(manager
->streamIsValid(type
));
101 class DeviceStreamManagerTest
: public ::testing::Test
106 TEST_F(DeviceStreamManagerTest
, CorrectStreamsAreReturnedOnNonbondedDevice
)
108 // It would be nice to test that the priority is high when it can
109 // be, but that requires calling the same API calls we're testing
110 // that we've called, so it is not very useful.
111 const bool useTiming
= false;
113 const auto& testDeviceList
= getTestHardwareEnvironment()->getTestDeviceList();
114 for (const auto& testDevice
: testDeviceList
)
116 const DeviceInformation
& deviceInfo
= testDevice
->deviceInfo();
117 setActiveDevice(deviceInfo
);
120 SCOPED_TRACE("No DD, no PME rank, no GPU update");
121 SimulationWorkload simulationWork
;
122 simulationWork
.useGpuPme
= false;
123 simulationWork
.useGpuPmePpCommunication
= false;
124 simulationWork
.useGpuUpdate
= false;
125 bool havePpDomainDecomposition
= false;
126 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
128 expectValidStreams(&manager
, { DeviceStreamType::NonBondedLocal
});
129 expectInvalidStreams(&manager
, { DeviceStreamType::NonBondedNonLocal
,
130 DeviceStreamType::Pme
, DeviceStreamType::PmePpTransfer
,
131 DeviceStreamType::UpdateAndConstraints
});
135 SCOPED_TRACE("With DD, no PME rank, no GPU update");
136 SimulationWorkload simulationWork
;
137 simulationWork
.useGpuPme
= false;
138 simulationWork
.useGpuPmePpCommunication
= false;
139 simulationWork
.useGpuUpdate
= false;
140 bool havePpDomainDecomposition
= true;
141 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
143 expectValidStreams(&manager
, { DeviceStreamType::NonBondedLocal
,
144 DeviceStreamType::NonBondedNonLocal
});
145 expectInvalidStreams(&manager
, { DeviceStreamType::Pme
, DeviceStreamType::PmePpTransfer
,
146 DeviceStreamType::UpdateAndConstraints
});
150 SCOPED_TRACE("No DD, with PME rank, no GPU update");
151 SimulationWorkload simulationWork
;
152 simulationWork
.useGpuPme
= true;
153 simulationWork
.useGpuPmePpCommunication
= true;
154 simulationWork
.useGpuUpdate
= false;
155 bool havePpDomainDecomposition
= false;
156 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
158 expectValidStreams(&manager
, { DeviceStreamType::Pme
, DeviceStreamType::NonBondedLocal
,
159 DeviceStreamType::PmePpTransfer
,
160 DeviceStreamType::UpdateAndConstraints
});
161 expectInvalidStreams(&manager
, { DeviceStreamType::NonBondedNonLocal
});
165 SCOPED_TRACE("With DD, with PME rank, no GPU update");
166 SimulationWorkload simulationWork
;
167 simulationWork
.useGpuPme
= true;
168 simulationWork
.useGpuPmePpCommunication
= true;
169 simulationWork
.useGpuUpdate
= false;
170 bool havePpDomainDecomposition
= true;
171 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
173 expectValidStreams(&manager
, { DeviceStreamType::Pme
, DeviceStreamType::NonBondedLocal
,
174 DeviceStreamType::NonBondedNonLocal
, DeviceStreamType::PmePpTransfer
,
175 DeviceStreamType::UpdateAndConstraints
});
179 SCOPED_TRACE("No DD, no PME rank, with GPU update");
180 SimulationWorkload simulationWork
;
181 simulationWork
.useGpuPme
= false;
182 simulationWork
.useGpuPmePpCommunication
= false;
183 simulationWork
.useGpuUpdate
= true;
184 bool havePpDomainDecomposition
= false;
185 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
187 expectValidStreams(&manager
, { DeviceStreamType::NonBondedLocal
,
188 DeviceStreamType::UpdateAndConstraints
});
189 expectInvalidStreams(&manager
, { DeviceStreamType::NonBondedNonLocal
,
190 DeviceStreamType::Pme
, DeviceStreamType::PmePpTransfer
});
194 SCOPED_TRACE("With DD, no PME rank, with GPU update");
195 SimulationWorkload simulationWork
;
196 simulationWork
.useGpuPme
= false;
197 simulationWork
.useGpuPmePpCommunication
= false;
198 simulationWork
.useGpuUpdate
= true;
199 bool havePpDomainDecomposition
= true;
200 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
202 expectValidStreams(&manager
, { DeviceStreamType::NonBondedLocal
, DeviceStreamType::NonBondedNonLocal
,
203 DeviceStreamType::UpdateAndConstraints
});
204 expectInvalidStreams(&manager
, { DeviceStreamType::Pme
, DeviceStreamType::PmePpTransfer
});
208 SCOPED_TRACE("No DD, with PME rank, with GPU update");
209 SimulationWorkload simulationWork
;
210 simulationWork
.useGpuPme
= true;
211 simulationWork
.useGpuPmePpCommunication
= true;
212 simulationWork
.useGpuUpdate
= true;
213 bool havePpDomainDecomposition
= false;
214 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
216 expectValidStreams(&manager
, { DeviceStreamType::Pme
, DeviceStreamType::NonBondedLocal
,
217 DeviceStreamType::PmePpTransfer
,
218 DeviceStreamType::UpdateAndConstraints
});
219 expectInvalidStreams(&manager
, { DeviceStreamType::NonBondedNonLocal
});
223 SCOPED_TRACE("With DD, with PME rank, with GPU update");
224 SimulationWorkload simulationWork
;
225 simulationWork
.useGpuPme
= true;
226 simulationWork
.useGpuPmePpCommunication
= true;
227 simulationWork
.useGpuUpdate
= true;
228 bool havePpDomainDecomposition
= true;
229 DeviceStreamManager
manager(deviceInfo
, havePpDomainDecomposition
, simulationWork
, useTiming
);
231 expectValidStreams(&manager
, { DeviceStreamType::Pme
, DeviceStreamType::NonBondedLocal
,
232 DeviceStreamType::NonBondedNonLocal
, DeviceStreamType::PmePpTransfer
,
233 DeviceStreamType::UpdateAndConstraints
});