From 814bffe4effd6951c65be9423d6ed7fef060e74e Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Mon, 7 May 2018 17:00:46 +0200 Subject: [PATCH] Remove commrec from set_ddbox() Change-Id: I62642ae8ed9eb8b839e577d8be904204d27a8d0d --- src/gromacs/domdec/domdec.cpp | 8 +++---- src/gromacs/domdec/domdec.h | 2 +- src/gromacs/domdec/domdec_box.cpp | 47 ++++++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp index 94a4f39565..bf55c41141 100644 --- a/src/gromacs/domdec/domdec.cpp +++ b/src/gromacs/domdec/domdec.cpp @@ -7289,7 +7289,7 @@ static gmx_bool test_dd_cutoff(t_commrec *cr, dd = cr->dd; - set_ddbox(dd, FALSE, cr, ir, state->box, + set_ddbox(dd, FALSE, ir, state->box, TRUE, &dd->comm->cgs_gl, as_rvec_array(state->x.data()), &ddbox); LocallyLimited = 0; @@ -9418,7 +9418,7 @@ void dd_partition_system(FILE *fplog, rvec *xGlobal = (SIMMASTER(cr) ? as_rvec_array(state_global->x.data()) : nullptr); - set_ddbox(dd, bMasterState, cr, ir, + set_ddbox(dd, bMasterState, ir, SIMMASTER(cr) ? state_global->box : nullptr, TRUE, cgs_gl, xGlobal, &ddbox); @@ -9476,7 +9476,7 @@ void dd_partition_system(FILE *fplog, dd_set_cginfo(dd->index_gl, 0, dd->ncg_home, fr, comm->bLocalCG); - set_ddbox(dd, bMasterState, cr, ir, state_local->box, + set_ddbox(dd, bMasterState, ir, state_local->box, TRUE, &top_local->cgs, as_rvec_array(state_local->x.data()), &ddbox); bRedist = isDlbOn(comm); @@ -9495,7 +9495,7 @@ void dd_partition_system(FILE *fplog, copy_rvec(comm->box0, ddbox.box0 ); copy_rvec(comm->box_size, ddbox.box_size); } - set_ddbox(dd, bMasterState, cr, ir, state_local->box, + set_ddbox(dd, bMasterState, ir, state_local->box, bNStGlobalComm, &top_local->cgs, as_rvec_array(state_local->x.data()), &ddbox); bBoxChanged = TRUE; diff --git a/src/gromacs/domdec/domdec.h b/src/gromacs/domdec/domdec.h index 662b56f3c7..18806ac240 100644 --- a/src/gromacs/domdec/domdec.h +++ b/src/gromacs/domdec/domdec.h @@ -466,7 +466,7 @@ real dd_choose_grid(FILE *fplog, /* In domdec_box.c */ /*! \brief Set the box and PBC data in \p ddbox */ -void set_ddbox(gmx_domdec_t *dd, gmx_bool bMasterState, const t_commrec *cr_sum, +void set_ddbox(gmx_domdec_t *dd, gmx_bool bMasterState, const t_inputrec *ir, const matrix box, gmx_bool bCalcUnboundedSize, const t_block *cgs, const rvec *x, gmx_ddbox_t *ddbox); diff --git a/src/gromacs/domdec/domdec_box.cpp b/src/gromacs/domdec/domdec_box.cpp index 5f931538c5..a907778921 100644 --- a/src/gromacs/domdec/domdec_box.cpp +++ b/src/gromacs/domdec/domdec_box.cpp @@ -44,6 +44,8 @@ #include "gmxpre.h" +#include "config.h" + #include "gromacs/domdec/domdec.h" #include "gromacs/domdec/domdec_network.h" #include "gromacs/domdec/domdec_struct.h" @@ -55,16 +57,16 @@ #include "gromacs/mdtypes/inputrec.h" #include "gromacs/pbcutil/pbc.h" #include "gromacs/topology/block.h" +#include "gromacs/utility/basedefinitions.h" #include "gromacs/utility/fatalerror.h" /*! \brief Calculates the average and standard deviation in 3D of n charge groups */ static void calc_cgcm_av_stddev(const t_block *cgs, int n, const rvec *x, rvec av, rvec stddev, - const t_commrec *cr_sum) + const MPI_Comm *mpiCommunicator) { int *cgindex; dvec s1, s2; - double buf[7]; int cg, d, k0, k1, k, nrcg; real inv_ncg; rvec cg_cm; @@ -103,22 +105,33 @@ static void calc_cgcm_av_stddev(const t_block *cgs, int n, const rvec *x, } } - if (cr_sum != nullptr) +#if GMX_MPI + if (mpiCommunicator) { - for (d = 0; d < DIM; d++) + constexpr int c_bufSize = 7; + double sendBuffer[c_bufSize]; + double receiveBuffer[c_bufSize]; + + for (int d = 0; d < DIM; d++) { - buf[d] = s1[d]; - buf[DIM+d] = s2[d]; + sendBuffer[d] = s1[d]; + sendBuffer[DIM + d] = s2[d]; } - buf[6] = n; - gmx_sumd(7, buf, cr_sum); - for (d = 0; d < DIM; d++) + sendBuffer[6] = n; + + MPI_Allreduce(sendBuffer, receiveBuffer, c_bufSize, MPI_DOUBLE, + MPI_SUM, *mpiCommunicator); + + for (int d = 0; d < DIM; d++) { - s1[d] = buf[d]; - s2[d] = buf[DIM+d]; + s1[d] = receiveBuffer[d]; + s2[d] = receiveBuffer[DIM + d]; } - n = (int)(buf[6] + 0.5); + n = (int)(receiveBuffer[6] + 0.5); } +#else // GMX_MPI + GMX_UNUSED_VALUE(mpiCommunicator); +#endif // GMX_MPI dsvmul(1.0/n, s1, s1); dsvmul(1.0/n, s2, s2); @@ -240,7 +253,7 @@ static void set_tric_dir(const ivec *dd_nc, gmx_ddbox_t *ddbox, const matrix box /*! \brief This function calculates bounding box and pbc info and populates ddbox */ static void low_set_ddbox(const t_inputrec *ir, const ivec *dd_nc, const matrix box, gmx_bool bCalcUnboundedSize, int ncg, const t_block *cgs, const rvec *x, - const t_commrec *cr_sum, + const MPI_Comm *mpiCommunicator, gmx_ddbox_t *ddbox) { rvec av, stddev; @@ -258,7 +271,7 @@ static void low_set_ddbox(const t_inputrec *ir, const ivec *dd_nc, const matrix if (ddbox->nboundeddim < DIM && bCalcUnboundedSize) { - calc_cgcm_av_stddev(cgs, ncg, x, av, stddev, cr_sum); + calc_cgcm_av_stddev(cgs, ncg, x, av, stddev, mpiCommunicator); /* GRID_STDDEV_FAC * stddev * gives a uniform load for a rectangular block of cg's. @@ -281,16 +294,18 @@ static void low_set_ddbox(const t_inputrec *ir, const ivec *dd_nc, const matrix set_tric_dir(dd_nc, ddbox, box); } -void set_ddbox(gmx_domdec_t *dd, gmx_bool bMasterState, const t_commrec *cr_sum, +void set_ddbox(gmx_domdec_t *dd, gmx_bool bMasterState, const t_inputrec *ir, const matrix box, gmx_bool bCalcUnboundedSize, const t_block *cgs, const rvec *x, gmx_ddbox_t *ddbox) { if (!bMasterState || DDMASTER(dd)) { + bool needToReduceCoordinateData = (!bMasterState && dd->nnodes > 1); + low_set_ddbox(ir, &dd->nc, box, bCalcUnboundedSize, bMasterState ? cgs->nr : dd->ncg_home, cgs, x, - bMasterState ? nullptr : cr_sum, + needToReduceCoordinateData ? &dd->mpi_comm_all : nullptr, ddbox); } -- 2.11.4.GIT