src/gromacs/domdec/dlbtiming.cpp

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2017, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35
  36 #include "gmxpre.h"
  37
  38 #include "dlbtiming.h"
  39
  40 #include "gromacs/domdec/domdec.h"
  41 #include "gromacs/utility/gmxassert.h"
  42
  43 #include "domdec_internal.h"
  44
  45 /*! \brief Struct for timing the region for dynamic load balancing */
  46 struct BalanceRegion
  47 {
  48     /*! \brief Constructor */
  49     BalanceRegion() :
  50         isOpen(false),
  51         isOpenOnCpu(false),
  52         isOpenOnGpu(false),
  53         cyclesOpenCpu(0),
  54         cyclesLastCpu(0)
  55     {
  56     }
  57
  58     bool         isOpen;         /**< Are we in an open balancing region? */
  59     bool         isOpenOnCpu;    /**< Is the, currently open, region still open on the CPU side? */
  60     bool         isOpenOnGpu;    /**< Is the, currently open, region open on the GPU side? */
  61     gmx_cycles_t cyclesOpenCpu;  /**< Cycle count when opening the CPU region */
  62     gmx_cycles_t cyclesLastCpu;  /**< Cycle count at the last call to \p ddCloseBalanceRegionCpu() */
  63 };
  64
  65 BalanceRegion *ddBalanceRegionAllocate()
  66 {
  67     return new BalanceRegion;
  68 }
  69
  70 /*! \brief Returns the pointer to the balance region.
  71  *
  72  * This should be replaced by a properly managed BalanceRegion class,
  73  * but that requires a lot of refactoring in domdec.cpp.
  74  */
  75 static BalanceRegion *getBalanceRegion(const gmx_domdec_t *dd)
  76 {
  77     GMX_ASSERT(dd != nullptr && dd->comm != nullptr, "Balance regions should only be used with DD");
  78     BalanceRegion *region = dd->comm->balanceRegion;
  79     GMX_ASSERT(region != nullptr, "Balance region should be initialized before use");
  80     return region;
  81 }
  82
  83 void ddOpenBalanceRegionCpu(const gmx_domdec_t                    *dd,
  84                             DdAllowBalanceRegionReopen gmx_unused  allowReopen)
  85 {
  86     BalanceRegion *reg = getBalanceRegion(dd);
  87     if (dd->comm->bRecordLoad)
  88     {
  89         GMX_ASSERT(allowReopen == DdAllowBalanceRegionReopen::yes || !reg->isOpen, "Should not open an already opened region");
  90
  91         reg->cyclesOpenCpu = gmx_cycles_read();
  92         reg->isOpen        = true;
  93         reg->isOpenOnCpu   = true;
  94         reg->isOpenOnGpu   = false;
  95     }
  96 }
  97
  98 void ddOpenBalanceRegionGpu(const gmx_domdec_t *dd)
  99 {
 100     BalanceRegion *reg = getBalanceRegion(dd);
 101     if (reg->isOpen)
 102     {
 103         GMX_ASSERT(!reg->isOpenOnGpu, "Can not re-open a GPU balance region");
 104         reg->isOpenOnGpu = true;
 105     }
 106 }
 107
 108 void ddReopenBalanceRegionCpu(const gmx_domdec_t *dd)
 109 {
 110     BalanceRegion *reg = getBalanceRegion(dd);
 111     /* If the GPU is busy, don't reopen as we are overlapping with work */
 112     if (reg->isOpen && !reg->isOpenOnGpu)
 113     {
 114         reg->cyclesOpenCpu = gmx_cycles_read();
 115     }
 116 }
 117
 118 void ddCloseBalanceRegionCpu(const gmx_domdec_t *dd)
 119 {
 120     BalanceRegion *reg = getBalanceRegion(dd);
 121     if (reg->isOpen && reg->isOpenOnCpu)
 122     {
 123         GMX_ASSERT(reg->isOpenOnCpu, "Can only close an open region");
 124         gmx_cycles_t cycles = gmx_cycles_read();
 125         reg->isOpenOnCpu    = false;
 126
 127         if (reg->isOpenOnGpu)
 128         {
 129             /* Store the cycles for estimating the GPU/CPU overlap time */
 130             reg->cyclesLastCpu = cycles;
 131         }
 132         else
 133         {
 134             /* We can close the region */
 135             float cyclesCpu   = cycles - reg->cyclesOpenCpu;
 136             dd_cycles_add(dd, cyclesCpu, ddCyclF);
 137             reg->isOpen       = false;
 138         }
 139     }
 140 }
 141
 142 void ddCloseBalanceRegionGpu(const gmx_domdec_t          *dd,
 143                              float                        waitGpuCyclesInCpuRegion,
 144                              DdBalanceRegionWaitedForGpu  waitedForGpu)
 145 {
 146     BalanceRegion *reg = getBalanceRegion(dd);
 147     if (reg->isOpen)
 148     {
 149         GMX_ASSERT(reg->isOpenOnGpu, "Can not close a non-open GPU balance region");
 150         GMX_ASSERT(!reg->isOpenOnCpu, "The GPU region should be closed after closing the CPU region");
 151
 152         float waitGpuCyclesEstimate = gmx_cycles_read() - reg->cyclesLastCpu;
 153         if (waitedForGpu == DdBalanceRegionWaitedForGpu::no)
 154         {
 155             /* The actual time could be anywhere between 0 and
 156              * waitCyclesEstimate. Using half is the best we can do.
 157              */
 158             const float unknownWaitEstimateFactor = 0.5f;
 159             waitGpuCyclesEstimate *= unknownWaitEstimateFactor;
 160         }
 161
 162         float cyclesCpu = reg->cyclesLastCpu - reg->cyclesOpenCpu;
 163         dd_cycles_add(dd, cyclesCpu + waitGpuCyclesEstimate, ddCyclF);
 164
 165         /* Register the total GPU wait time, to redistribute with GPU sharing */
 166         dd_cycles_add(dd, waitGpuCyclesInCpuRegion + waitGpuCyclesEstimate, ddCyclWaitGPU);
 167
 168         /* Close the region */
 169         reg->isOpenOnGpu = false;
 170         reg->isOpen      = false;
 171     }
 172 }