From 9648979634bcca0014299f05ce46ecf83dc4b01d Mon Sep 17 00:00:00 2001
From: Berk Hess <hess@kth.se>
Date: Sat, 25 Nov 2017 09:57:53 +0100
Subject: [PATCH] Enable auto thread pinning with thread limiting

Recently the possibility of automated limiting the number of OpenMP
threads was introduced (with PME on the GPU). Note that this could
already happen with very small systems. When not using all hardware
threads, pinning would be disabled and a warning was issued.
Now pinning is enabled an no warnings are issued when the user did
not specify any thread and pinning settings.

Changed "threads" in the mdrun -ntmpi description to "ranks".

Change-Id: I5f5688a4e2d35fdddbae3aeffae256158a13da5c
---
 src/gromacs/hardware/hw_info.h                        |  2 ++
 src/gromacs/mdrunutility/tests/threadaffinitytest.cpp |  3 ++-
 src/gromacs/mdrunutility/threadaffinity.cpp           | 11 +++++++----
 src/gromacs/taskassignment/resourcedivision.cpp       | 10 ++++++++++
 src/programs/mdrun/mdrun.cpp                          |  2 +-
 5 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/src/gromacs/hardware/hw_info.h b/src/gromacs/hardware/hw_info.h
index e86d8cdc4b..c179de6812 100644
--- a/src/gromacs/hardware/hw_info.h
+++ b/src/gromacs/hardware/hw_info.h
@@ -116,6 +116,8 @@ struct gmx_hw_opt_t
     std::string   gpuIdsAvailable = "";
     //! Empty, or a string provided by the user mapping GPU tasks to devices.
     std::string   userGpuTaskAssignment = "";
+    //! Tells whether mdrun is free to choose the total number of threads (by choosing the number of OpenMP and/or thread-MPI threads).
+    bool          totNumThreadsIsAuto;
 };
 
 #endif
diff --git a/src/gromacs/mdrunutility/tests/threadaffinitytest.cpp b/src/gromacs/mdrunutility/tests/threadaffinitytest.cpp
index e9ee864fa3..862322a5a9 100644
--- a/src/gromacs/mdrunutility/tests/threadaffinitytest.cpp
+++ b/src/gromacs/mdrunutility/tests/threadaffinitytest.cpp
@@ -75,7 +75,8 @@ ThreadAffinityTestHelper::ThreadAffinityTestHelper()
 #if GMX_MPI
     cr_->mpi_comm_mysim = MPI_COMM_WORLD;
 #endif
-    hwOpt_.thread_affinity = threadaffAUTO;
+    hwOpt_.thread_affinity     = threadaffAUTO;
+    hwOpt_.totNumThreadsIsAuto = false;
 }
 
 ThreadAffinityTestHelper::~ThreadAffinityTestHelper()
diff --git a/src/gromacs/mdrunutility/threadaffinity.cpp b/src/gromacs/mdrunutility/threadaffinity.cpp
index a4370e9450..c147b9de1d 100644
--- a/src/gromacs/mdrunutility/threadaffinity.cpp
+++ b/src/gromacs/mdrunutility/threadaffinity.cpp
@@ -115,7 +115,7 @@ get_thread_affinity_layout(const gmx::MDLogger &mdlog,
                            const t_commrec *cr,
                            const gmx::HardwareTopology &hwTop,
                            int   threads,
-                           bool  automatic,
+                           bool  affinityIsAutoAndNumThreadsIsNotAuto,
                            int pin_offset, int * pin_stride,
                            int **localityOrder)
 {
@@ -176,7 +176,7 @@ get_thread_affinity_layout(const gmx::MDLogger &mdlog,
     }
     bool validLayout = !invalidValue;
 
-    if (automatic)
+    if (affinityIsAutoAndNumThreadsIsNotAuto)
     {
         invalidValue = (threads != hwThreads);
         bool warn = (invalidValue && threads > 1 && threads < hwThreads);
@@ -426,9 +426,12 @@ gmx_set_thread_affinity(const gmx::MDLogger         &mdlog,
         GMX_LOG(mdlog.warning).appendTextFormatted("Applying core pinning offset %d", offset);
     }
 
-    bool automatic = (hw_opt->thread_affinity == threadaffAUTO);
+    bool affinityIsAutoAndNumThreadsIsNotAuto =
+        (hw_opt->thread_affinity == threadaffAUTO &&
+         !hw_opt->totNumThreadsIsAuto);
     bool validLayout
-        = get_thread_affinity_layout(mdlog, cr, hwTop, nthread_node, automatic,
+        = get_thread_affinity_layout(mdlog, cr, hwTop, nthread_node,
+                                     affinityIsAutoAndNumThreadsIsNotAuto,
                                      offset, &core_pinning_stride, &localityOrder);
     const gmx::sfree_guard  localityOrderGuard(localityOrder);
 
diff --git a/src/gromacs/taskassignment/resourcedivision.cpp b/src/gromacs/taskassignment/resourcedivision.cpp
index 253e2b54bf..6b470a377d 100644
--- a/src/gromacs/taskassignment/resourcedivision.cpp
+++ b/src/gromacs/taskassignment/resourcedivision.cpp
@@ -716,6 +716,16 @@ void check_and_update_hw_opt_1(gmx_hw_opt_t    *hw_opt,
     }
 #endif
 
+    /* With thread-MPI the master thread sets hw_opt->totNumThreadsIsAuto.
+     * The other threads receive a partially processed hw_opt from the master
+     * thread and should not set hw_opt->totNumThreadsIsAuto again.
+     */
+    if (!GMX_THREAD_MPI || SIMMASTER(cr))
+    {
+        /* Check if mdrun is free to choose the total number of threads */
+        hw_opt->totNumThreadsIsAuto = (hw_opt->nthreads_omp == 0 && hw_opt->nthreads_omp_pme == 0 && hw_opt->nthreads_tot == 0);
+    }
+
     if (bHasOmpSupport)
     {
         /* Check restrictions on PME thread related options set by the user */
diff --git a/src/programs/mdrun/mdrun.cpp b/src/programs/mdrun/mdrun.cpp
index 0c131c9a16..482a0d9b78 100644
--- a/src/programs/mdrun/mdrun.cpp
+++ b/src/programs/mdrun/mdrun.cpp
@@ -271,7 +271,7 @@ int Mdrunner::mainFunction(int argc, char *argv[])
         { "-nt",      FALSE, etINT, {&hw_opt.nthreads_tot},
           "Total number of threads to start (0 is guess)" },
         { "-ntmpi",   FALSE, etINT, {&hw_opt.nthreads_tmpi},
-          "Number of thread-MPI threads to start (0 is guess)" },
+          "Number of thread-MPI ranks to start (0 is guess)" },
         { "-ntomp",   FALSE, etINT, {&hw_opt.nthreads_omp},
           "Number of OpenMP threads per MPI rank to start (0 is guess)" },
         { "-ntomp_pme", FALSE, etINT, {&hw_opt.nthreads_omp_pme},
-- 
2.11.4.GIT