From d6d0678cd7dcf72ecc2b6189e733236c7cd135e1 Mon Sep 17 00:00:00 2001
From: Jaemin Choi <jchoi157@illinois.edu>
Date: Tue, 25 Sep 2018 01:05:45 -0400
Subject: [PATCH] Update references to hapi_src to hapi_impl and revert
 hapiRegisterCallbacks

Change-Id: Ic825715c76058c21e73558a1cf8dc154c218df7e
---
 src/arch/common/special.sh           |  2 +-
 src/arch/cuda/hybridAPI/Makefile     |  4 ++--
 src/arch/cuda/hybridAPI/README.txt   |  4 ++--
 src/arch/cuda/hybridAPI/hapi_impl.cu | 22 ++++++++++++----------
 src/arch/cuda/hybridAPI/hapi_impl.h  |  6 +++---
 src/ck-core/init.C                   |  2 +-
 6 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/arch/common/special.sh b/src/arch/common/special.sh
index eda97272e8..bce585caf4 100755
--- a/src/arch/common/special.sh
+++ b/src/arch/common/special.sh
@@ -13,7 +13,7 @@ then
   ./gathertree $SRCBASE/arch/cuda .
 
 # make links
-  test ! -f "../include/hapi_src.h" && ./system_ln "../tmp/hybridAPI/hapi_src.h" ../include && test ! -f "../include/hapi.h" && ./system_ln "../tmp/hybridAPI/hapi.h" ../include && test ! -f "../include/hapi_nvtx.h" && ./system_ln "../tmp/hybridAPI/hapi_nvtx.h" ../include
+  test ! -f "../include/hapi_impl.h" && ./system_ln "../tmp/hybridAPI/hapi_impl.h" ../include && test ! -f "../include/hapi.h" && ./system_ln "../tmp/hybridAPI/hapi.h" ../include && test ! -f "../include/hapi_nvtx.h" && ./system_ln "../tmp/hybridAPI/hapi_nvtx.h" ../include
 
 #make library
   export CHARMINC=../include
diff --git a/src/arch/cuda/hybridAPI/Makefile b/src/arch/cuda/hybridAPI/Makefile
index a5f28e11ff..cda563cd95 100644
--- a/src/arch/cuda/hybridAPI/Makefile
+++ b/src/arch/cuda/hybridAPI/Makefile
@@ -17,8 +17,8 @@ libcudahybridapi.a: hybridapi.o
 	-rm -f $@
 	ar q $@ hybridapi.o
 
-hybridapi.o: hapi_src.cu hapi_src.h hapi.h hapi_nvtx.h
-	$(NVCC) $(NVCC_FLAGS) $(NVCC_INC) -o hybridapi.o hapi_src.cu
+hybridapi.o: hapi_impl.cu hapi_impl.h hapi.h hapi_nvtx.h
+	$(NVCC) $(NVCC_FLAGS) $(NVCC_INC) -o hybridapi.o hapi_impl.cu
 
 clean:
 	rm -f *.o *.a
diff --git a/src/arch/cuda/hybridAPI/README.txt b/src/arch/cuda/hybridAPI/README.txt
index 5ded5f559f..bb07d255d1 100644
--- a/src/arch/cuda/hybridAPI/README.txt
+++ b/src/arch/cuda/hybridAPI/README.txt
@@ -65,10 +65,10 @@ and is set as the default.
   Contains functions that utilize NVIDIA's NVTX feature.
   To be included in user's CUDA source code.
 
-<hapi_src.h>
+<hapi_impl.h>
   Contains function declarations used by the Charm++ runtime.
 
-<hapi_src.cu>
+<hapi_impl.cu>
   Contains the actual implementation of HAPI functionalities.
 
 4. NVCC Flags
diff --git a/src/arch/cuda/hybridAPI/hapi_impl.cu b/src/arch/cuda/hybridAPI/hapi_impl.cu
index 8422fcf09b..dd9f025d5a 100644
--- a/src/arch/cuda/hybridAPI/hapi_impl.cu
+++ b/src/arch/cuda/hybridAPI/hapi_impl.cu
@@ -1,5 +1,5 @@
 #include "hapi.h"
-#include "hapi_src.h"
+#include "hapi_impl.h"
 #include "converse.h"
 #include "ckcallback.h"
 #include <stdio.h>
@@ -537,15 +537,17 @@ static void* lightCallback(void *arg) {
 }
 
 // Register callback functions. All PEs need to call this.
-void registerCallbacks() {
-  CmiAssignOnce(&CsvAccess(gpu_manager).host_to_device_cb_idx_,
-      CmiRegisterHandler((CmiHandler)hostToDeviceCallback));
-  CmiAssignOnce(&CsvAccess(gpu_manager).kernel_cb_idx_,
-      CmiRegisterHandler((CmiHandler)kernelCallback));
-  CmiAssignOnce(&CsvAccess(gpu_manager).device_to_host_cb_idx_,
-      CmiRegisterHandler((CmiHandler)deviceToHostCallback));
-  CmiAssignOnce(&CsvAccess(gpu_manager).light_cb_idx_,
-      CmiRegisterHandler((CmiHandler)lightCallback));
+void hapiRegisterCallbacks() {
+  // FIXME: Potential race condition on assignments, but CmiAssignOnce
+  // causes a hang at startup.
+  CsvAccess(gpu_manager).host_to_device_cb_idx_
+    = CmiRegisterHandler((CmiHandler)hostToDeviceCallback);
+  CsvAccess(gpu_manager).kernel_cb_idx_
+    = CmiRegisterHandler((CmiHandler)kernelCallback);
+  CsvAccess(gpu_manager).device_to_host_cb_idx_
+    = CmiRegisterHandler((CmiHandler)deviceToHostCallback);
+  CsvAccess(gpu_manager).light_cb_idx_
+    = CmiRegisterHandler((CmiHandler)lightCallback);
 }
 
 // Callback function invoked by the CUDA runtime certain parts of GPU work are
diff --git a/src/arch/cuda/hybridAPI/hapi_impl.h b/src/arch/cuda/hybridAPI/hapi_impl.h
index 878680a831..3c30abdfc0 100644
--- a/src/arch/cuda/hybridAPI/hapi_impl.h
+++ b/src/arch/cuda/hybridAPI/hapi_impl.h
@@ -1,5 +1,5 @@
-#ifndef __HAPI_SRC_H_
-#define __HAPI_SRC_H_
+#ifndef __HAPI_IMPL_H_
+#define __HAPI_IMPL_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -41,4 +41,4 @@ typedef struct _bufferPool {
 }
 #endif
 
-#endif // __HAPI_SRC_H_
+#endif // __HAPI_IMPL_H_
diff --git a/src/ck-core/init.C b/src/ck-core/init.C
index 5e44fe8f92..f0789a0bec 100644
--- a/src/ck-core/init.C
+++ b/src/ck-core/init.C
@@ -77,7 +77,7 @@ never be excluded...
 #endif
 
 #if CMK_CUDA
-#include "hapi_src.h"
+#include "hapi_impl.h"
 #endif
 
 void CkRestartMain(const char* dirname, CkArgMsg* args);
-- 
2.11.4.GIT