From 7282084b44612424876c53adff1e1563e5dfc83d Mon Sep 17 00:00:00 2001 From: Mark Abraham Date: Tue, 8 Aug 2017 02:39:34 +0200 Subject: [PATCH] Parse user-supplied GPU task assignment only when needed Now there is never both a gpuTaskAssignment and a userGpuTaskAssignment present, which could have been misused or misunderstood. Also fixed somewhat related outdataed naming and documentation for gmx_count_gpu_dev_unique(). Change-Id: I09d27857b6a91a49735850655bdf26af4bd88445 --- src/gromacs/hardware/hardwareassign.cpp | 6 ++++-- src/gromacs/hardware/hardwareassign.h | 9 +++++---- src/gromacs/hardware/printhardware.cpp | 10 +++++----- src/programs/mdrun/resource-division.cpp | 4 +++- src/programs/mdrun/resource-division.h | 1 - src/programs/mdrun/runner.cpp | 14 +++++--------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/gromacs/hardware/hardwareassign.cpp b/src/gromacs/hardware/hardwareassign.cpp index bd9d3fc50e..fda0b96c69 100644 --- a/src/gromacs/hardware/hardwareassign.cpp +++ b/src/gromacs/hardware/hardwareassign.cpp @@ -47,6 +47,7 @@ #include "gromacs/gmxlib/network.h" #include "gromacs/gpu_utils/gpu_utils.h" #include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/hw_info.h" #include "gromacs/mdtypes/commrec.h" #include "gromacs/utility/basenetwork.h" #include "gromacs/utility/exceptions.h" @@ -227,7 +228,7 @@ static std::vector getCompatibleGpus(const gmx_gpu_info_t &gpu_info) std::vector mapPpRanksToGpus(bool rankCanUseGpu, const t_commrec *cr, const gmx_gpu_info_t &gpu_info, - const std::vector &userGpuTaskAssignment) + const gmx_hw_opt_t &hw_opt) { std::vector taskAssignment; @@ -237,8 +238,9 @@ std::vector mapPpRanksToGpus(bool rankCanUseGpu, } auto compatibleGpus = getCompatibleGpus(gpu_info); - if (!userGpuTaskAssignment.empty()) + if (!hw_opt.gpuIdTaskAssignment.empty()) { + auto userGpuTaskAssignment = parseGpuTaskAssignment(hw_opt.gpuIdTaskAssignment); exitUnlessUserGpuTaskAssignmentIsValid(gpu_info, compatibleGpus, userGpuTaskAssignment); taskAssignment = userGpuTaskAssignment; } diff --git a/src/gromacs/hardware/hardwareassign.h b/src/gromacs/hardware/hardwareassign.h index e20580c958..f5ac536121 100644 --- a/src/gromacs/hardware/hardwareassign.h +++ b/src/gromacs/hardware/hardwareassign.h @@ -41,6 +41,7 @@ #include "gromacs/utility/basedefinitions.h" struct gmx_gpu_info_t; +struct gmx_hw_opt_t; struct t_commrec; namespace gmx @@ -62,8 +63,8 @@ std::vector parseGpuTaskAssignment(const std::string &gpuTaskAssignment); * * Will return a validated mapping from PP ranks (ie tasks that can * run on GPUs) to the device IDs of compatible GPUs on their node. - * This will be the \c userGpuTaskAssignment if non-empty, otherwise - * a default automated mapping is generated. + * This will be from any non-empty assignment in hw_opt, otherwise a + * default automated mapping is generated. * * Note that PME-only ranks have always ignored mdrun -gpu_id, so do * not attempt to validate -gpu_id. They should continue this behaviour @@ -72,14 +73,14 @@ std::vector parseGpuTaskAssignment(const std::string &gpuTaskAssignment); * \param[in] rankCanUseGpu Whether this rank can execute a task on a GPU. * \param[in] cr Communication record. * \param[in] gpu_info Information detected about GPUs, including compatibility. - * \param[in] userGpuTaskAssignment Any GPU IDs required by the user. + * \param[in] hw_opt Parallelisation options, including any user-specified GPU task assignment. * * \returns A valid GPU selection. */ std::vector mapPpRanksToGpus(bool rankCanUseGpu, const t_commrec *cr, const gmx_gpu_info_t &gpu_info, - const std::vector &userGpuTaskAssignment); + const gmx_hw_opt_t &hw_opt); } // namespace diff --git a/src/gromacs/hardware/printhardware.cpp b/src/gromacs/hardware/printhardware.cpp index d9291f6aea..b2aa0a05fc 100644 --- a/src/gromacs/hardware/printhardware.cpp +++ b/src/gromacs/hardware/printhardware.cpp @@ -94,14 +94,14 @@ static int gmx_count_gpu_dev_shared(const std::vector &gpuTaskAssignment, /* Count and return the number of unique GPUs (per node) selected. * - * As sharing GPUs among multiple PP ranks is possible when the user passes - * GPU IDs, the number of GPUs user (per node) can be different from the - * number of GPU IDs selected. + * As sharing GPUs among multiple PP ranks is possible, the number of + * GPUs used (per node) can be different from the number of GPU IDs + * used. */ -static size_t gmx_count_gpu_dev_unique(const std::vector &userGpuTaskAssignment) +static size_t gmx_count_gpu_dev_unique(const std::vector &gpuTaskAssignment) { std::set uniqIds; - for (const auto &deviceId : userGpuTaskAssignment) + for (const auto &deviceId : gpuTaskAssignment) { uniqIds.insert(deviceId); } diff --git a/src/programs/mdrun/resource-division.cpp b/src/programs/mdrun/resource-division.cpp index b7b9afc38a..ce76012598 100644 --- a/src/programs/mdrun/resource-division.cpp +++ b/src/programs/mdrun/resource-division.cpp @@ -46,6 +46,7 @@ #include "gromacs/hardware/cpuinfo.h" #include "gromacs/hardware/detecthardware.h" +#include "gromacs/hardware/hardwareassign.h" #include "gromacs/hardware/hardwaretopology.h" #include "gromacs/hardware/hw_info.h" #include "gromacs/mdlib/gmx_omp_nthreads.h" @@ -337,7 +338,6 @@ class SingleRankChecker */ int get_nthreads_mpi(const gmx_hw_info_t *hwinfo, gmx_hw_opt_t *hw_opt, - const std::vector &userGpuTaskAssignment, const t_inputrec *inputrec, const gmx_mtop_t *mtop, const gmx::MDLogger &mdlog, @@ -349,6 +349,8 @@ int get_nthreads_mpi(const gmx_hw_info_t *hwinfo, const gmx::CpuInfo &cpuInfo = *hwinfo->cpuInfo; const gmx::HardwareTopology &hwTop = *hwinfo->hardwareTopology; + auto userGpuTaskAssignment = gmx::parseGpuTaskAssignment(hw_opt->gpuIdTaskAssignment); + /* TODO Here we handle the case where the user set GPU IDs, and further below we handle the case where the algorithm does not support multiple ranks. We need also to handle the case where diff --git a/src/programs/mdrun/resource-division.h b/src/programs/mdrun/resource-division.h index ff443fde19..14e07f23de 100644 --- a/src/programs/mdrun/resource-division.h +++ b/src/programs/mdrun/resource-division.h @@ -63,7 +63,6 @@ class MDLogger; */ int get_nthreads_mpi(const gmx_hw_info_t *hwinfo, gmx_hw_opt_t *hw_opt, - const std::vector &userGpuSelection, const t_inputrec *inputrec, const gmx_mtop_t *mtop, const gmx::MDLogger &mdlog, diff --git a/src/programs/mdrun/runner.cpp b/src/programs/mdrun/runner.cpp index 088a573540..5a71b61c27 100644 --- a/src/programs/mdrun/runner.cpp +++ b/src/programs/mdrun/runner.cpp @@ -721,13 +721,12 @@ int Mdrunner::mdrunner() * with things like whether support is compiled, or tMPI thread * count. */ bool emulateGpu = getenv("GMX_EMULATE_GPU") != nullptr; - auto userGpuTaskAssignment = parseGpuTaskAssignment(hw_opt.gpuIdTaskAssignment); bool forceUseCpu = (strncmp(nbpu_opt, "cpu", 3) == 0); - if (!userGpuTaskAssignment.empty() && forceUseCpu) + if (!hw_opt.gpuIdTaskAssignment.empty() && forceUseCpu) { gmx_fatal(FARGS, "GPU IDs were specified, and short-ranged interactions were assigned to the CPU. Make no more than one of these choices."); } - bool forceUsePhysicalGpu = (strncmp(nbpu_opt, "gpu", 3) == 0) || !userGpuTaskAssignment.empty(); + bool forceUsePhysicalGpu = (strncmp(nbpu_opt, "gpu", 3) == 0) || !hw_opt.gpuIdTaskAssignment.empty(); bool tryUsePhysicalGpu = (strncmp(nbpu_opt, "auto", 4) == 0) && !emulateGpu && (GMX_GPU != GMX_GPU_NONE); // Here we assume that SIMMASTER(cr) does not change even after the @@ -844,7 +843,6 @@ int Mdrunner::mdrunner() // Determine how many thread-MPI ranks to start. hw_opt.nthreads_tmpi = get_nthreads_mpi(hwinfo, &hw_opt, - userGpuTaskAssignment, inputrec, mtop, mdlog, doMembed); @@ -1110,17 +1108,15 @@ int Mdrunner::mdrunner() * or sharing devices on a node, either from the user * selection, or automatically. */ bool rankCanUseGpu = cr->duty & DUTY_PP; - gpuTaskAssignment = mapPpRanksToGpus(rankCanUseGpu, cr, hwinfo->gpu_info, - userGpuTaskAssignment); + gpuTaskAssignment = mapPpRanksToGpus(rankCanUseGpu, cr, hwinfo->gpu_info, hw_opt); } - reportGpuUsage(mdlog, hwinfo->gpu_info, !userGpuTaskAssignment.empty(), + reportGpuUsage(mdlog, hwinfo->gpu_info, !hw_opt.gpuIdTaskAssignment.empty(), gpuTaskAssignment, cr->nrank_pp_intranode, cr->nnodes > 1); /* check consistency across ranks of things like SIMD * support and number of GPUs selected */ - gmx_check_hw_runconf_consistency(mdlog, hwinfo, cr, hw_opt, !userGpuTaskAssignment.empty(), gpuTaskAssignment); - /* From now on, the userGpuTaskAssignment should never be used */ + gmx_check_hw_runconf_consistency(mdlog, hwinfo, cr, hw_opt, !hw_opt.gpuIdTaskAssignment.empty(), gpuTaskAssignment); /* Prevent other ranks from continuing after an inconsistency was found. * -- 2.11.4.GIT