From c946ef0722206315c48fe7924e0e2a01abfd708d Mon Sep 17 00:00:00 2001 From: Paul Bauer Date: Thu, 5 Nov 2020 12:35:58 +0100 Subject: [PATCH] Allow CUDA build tests to succeed without GPU Added early return to haloexchange GPU pathway test to not try to run the tests without a GPU (i.e. device list is empty). Refs #3789 --- src/gromacs/domdec/tests/haloexchange_mpi.cpp | 60 +++++++++++++++++++-------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/src/gromacs/domdec/tests/haloexchange_mpi.cpp b/src/gromacs/domdec/tests/haloexchange_mpi.cpp index 224d54e934..2d6e04a882 100644 --- a/src/gromacs/domdec/tests/haloexchange_mpi.cpp +++ b/src/gromacs/domdec/tests/haloexchange_mpi.cpp @@ -117,9 +117,11 @@ void initHaloData(RVec* x, const int numHomeAtoms, const int numAtomsTotal) * \param [in] h_x Atom coordinate data array on host * \param [in] numAtomsTotal Total number of atoms, including halo */ -void gpuHalo(gmx_domdec_t* dd, matrix box, RVec* h_x, int numAtomsTotal) +void gpuHalo(gmx_domdec_t* dd, matrix box, HostVector* h_x, int numAtomsTotal) { #if (GMX_GPU_CUDA && GMX_THREAD_MPI) + // pin memory if possible + changePinningPolicy(h_x, PinningPolicy::PinnedIfSupported); // Set up GPU hardware environment and assign this MPI rank to a device int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -135,7 +137,7 @@ void gpuHalo(gmx_domdec_t* dd, matrix box, RVec* h_x, int numAtomsTotal) int d_x_size_alloc = -1; reallocateDeviceBuffer(&d_x, numAtomsTotal, &d_x_size, &d_x_size_alloc, deviceContext); - copyToDeviceBuffer(&d_x, h_x, 0, numAtomsTotal, deviceStream, GpuApiCallBehavior::Sync, nullptr); + copyToDeviceBuffer(&d_x, h_x->data(), 0, numAtomsTotal, deviceStream, GpuApiCallBehavior::Sync, nullptr); GpuEventSynchronizer coordinatesReadyOnDeviceEvent; coordinatesReadyOnDeviceEvent.markEvent(deviceStream); @@ -157,7 +159,8 @@ void gpuHalo(gmx_domdec_t* dd, matrix box, RVec* h_x, int numAtomsTotal) haloCompletedEvent.waitForEvent(); // Copy results back to host - copyFromDeviceBuffer(h_x, &d_x, 0, numAtomsTotal, deviceStream, GpuApiCallBehavior::Sync, nullptr); + copyFromDeviceBuffer(h_x->data(), &d_x, 0, numAtomsTotal, deviceStream, + GpuApiCallBehavior::Sync, nullptr); freeDeviceBuffer(d_x); #else @@ -502,7 +505,6 @@ TEST(HaloExchangeTest, Coordinates1dHaloWith1Pulse) const int numHaloAtoms = 2; const int numAtomsTotal = numHomeAtoms + numHaloAtoms; HostVector h_x; - changePinningPolicy(&h_x, PinningPolicy::PinnedIfSupported); h_x.resize(numAtomsTotal); initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); @@ -533,11 +535,17 @@ TEST(HaloExchangeTest, Coordinates1dHaloWith1Pulse) if (GMX_GPU_CUDA && GMX_THREAD_MPI) // repeat with GPU halo codepath { + // early return if no devices are available. + if (getTestHardwareEnvironment()->getTestDeviceList().empty()) + { + return; + } + // Re-initialize input initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); // Perform GPU halo exchange - gpuHalo(&dd, box, h_x.data(), numAtomsTotal); + gpuHalo(&dd, box, &h_x, numAtomsTotal); // Check results checkResults1dHaloWith1Pulse(h_x.data(), &dd, numHomeAtoms); @@ -553,7 +561,6 @@ TEST(HaloExchangeTest, Coordinates1dHaloWith2Pulses) const int numHaloAtoms = 5; const int numAtomsTotal = numHomeAtoms + numHaloAtoms; HostVector h_x; - changePinningPolicy(&h_x, PinningPolicy::PinnedIfSupported); h_x.resize(numAtomsTotal); initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); @@ -584,11 +591,17 @@ TEST(HaloExchangeTest, Coordinates1dHaloWith2Pulses) if (GMX_GPU_CUDA && GMX_THREAD_MPI) // repeat with GPU halo codepath { + // early return if no devices are available. + if (getTestHardwareEnvironment()->getTestDeviceList().empty()) + { + return; + } + // Re-initialize input initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); // Perform GPU halo exchange - gpuHalo(&dd, box, h_x.data(), numAtomsTotal); + gpuHalo(&dd, box, &h_x, numAtomsTotal); // Check results checkResults1dHaloWith2Pulses(h_x.data(), &dd, numHomeAtoms); @@ -605,7 +618,6 @@ TEST(HaloExchangeTest, Coordinates2dHaloWith1PulseInEachDim) const int numHaloAtoms = 4; const int numAtomsTotal = numHomeAtoms + numHaloAtoms; HostVector h_x; - changePinningPolicy(&h_x, PinningPolicy::PinnedIfSupported); h_x.resize(numAtomsTotal); initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); @@ -636,11 +648,17 @@ TEST(HaloExchangeTest, Coordinates2dHaloWith1PulseInEachDim) if (GMX_GPU_CUDA && GMX_THREAD_MPI) // repeat with GPU halo codepath { + // early return if no devices are available. + if (getTestHardwareEnvironment()->getTestDeviceList().empty()) + { + return; + } + // Re-initialize input initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); // Perform GPU halo exchange - gpuHalo(&dd, box, h_x.data(), numAtomsTotal); + gpuHalo(&dd, box, &h_x, numAtomsTotal); // Check results checkResults2dHaloWith1PulseInEachDim(h_x.data(), &dd, numHomeAtoms); @@ -656,7 +674,6 @@ TEST(HaloExchangeTest, Coordinates2dHaloWith2PulsesInDim1) const int numHaloAtoms = 7; const int numAtomsTotal = numHomeAtoms + numHaloAtoms; HostVector h_x; - changePinningPolicy(&h_x, PinningPolicy::PinnedIfSupported); h_x.resize(numAtomsTotal); initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); @@ -685,16 +702,23 @@ TEST(HaloExchangeTest, Coordinates2dHaloWith2PulsesInDim1) // Check results checkResults2dHaloWith2PulsesInDim1(h_x.data(), &dd, numHomeAtoms); -#if (GMX_GPU_CUDA && GMX_THREAD_MPI) // repeat with GPU halo codepath - // Re-initialize input - initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); + if (GMX_GPU_CUDA && GMX_THREAD_MPI) // repeat with GPU halo codepath + { + // early return if no devices are available. + if (getTestHardwareEnvironment()->getTestDeviceList().empty()) + { + return; + } - // Perform GPU halo exchange - gpuHalo(&dd, box, h_x.data(), numAtomsTotal); + // Re-initialize input + initHaloData(h_x.data(), numHomeAtoms, numAtomsTotal); - // Check results - checkResults2dHaloWith2PulsesInDim1(h_x.data(), &dd, numHomeAtoms); -#endif + // Perform GPU halo exchange + gpuHalo(&dd, box, &h_x, numAtomsTotal); + + // Check results + checkResults2dHaloWith2PulsesInDim1(h_x.data(), &dd, numHomeAtoms); + } } } // namespace -- 2.11.4.GIT