Merge pull request #66 from mkstoyanov/better_gpu_aware_selection

improved the gpu-aware option

CMakeLists.txt

@@ -22,6 +22,16 @@ option(Heffte_ENABLE_TRACING  "Enable the tracing capabilities"  OFF)
 
 option(BUILD_SHARED_LIBS      "Builds shared libraries using CMake conventions"  ON)
 
+if (Heffte_ENABLE_CUDA OR Heffte_ENABLE_ROCM OR Heffte_ENABLE_ONEAPI)
+    if (Heffte_DISABLE_GPU_AWARE_MPI)
+        option(Heffte_ENABLE_GPU_AWARE_MPI "GPU to GPU direct MPI calls" OFF)
+    else()
+        option(Heffte_ENABLE_GPU_AWARE_MPI "GPU to GPU direct MPI calls" ON)
+    endif()
+else()
+    set(Heffte_ENABLE_GPU_AWARE_MPI OFF CACHE BOOL "GPU to GPU direct MPI calls" FORCE)
+endif()
+
 if (${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME})
     option(Heffte_ENABLE_TESTING "Enable the internal unit tests" ON)
 else()

cmake/HeffteConfig.cmake

@@ -116,10 +116,10 @@ set(Heffte_CUDA_FOUND    "@Heffte_ENABLE_CUDA@")
 set(Heffte_ROCM_FOUND    "@Heffte_ENABLE_ROCM@")
 set(Heffte_ONEAPI_FOUND  "@Heffte_ENABLE_ONEAPI@")
 set(Heffte_INTRINSICS_FOUND  "@Heffte_ENABLE_INTRINSICS@")
-if ("@Heffte_DISABLE_GPU_AWARE_MPI@")
-    set(Heffte_GPUAWARE_FOUND "OFF")
-else()
+if ("@Heffte_ENABLE_GPU_AWARE_MPI@")
     set(Heffte_GPUAWARE_FOUND "ON")
+else()
+    set(Heffte_GPUAWARE_FOUND "OFF")
 endif()
 
 check_required_components(Heffte)

cmake/print_summary.cmake

@@ -3,7 +3,7 @@ message(STATUS "")
 message(STATUS "heFFTe ${PROJECT_VERSION}")
 
 string(TOUPPER "${CMAKE_BUILD_TYPE}" HEFFTE_BUILD_TYPE)
-set(HEFFTE_OPTIONS "CMAKE_INSTALL_PREFIX;BUILD_SHARED_LIBS;CMAKE_BUILD_TYPE;CMAKE_CXX_FLAGS_${HEFFTE_BUILD_TYPE};CMAKE_CXX_FLAGS;MPI_CXX_COMPILER;MPI_CXX_COMPILE_OPTIONS")
+set(HEFFTE_OPTIONS "CMAKE_INSTALL_PREFIX;BUILD_SHARED_LIBS;CMAKE_BUILD_TYPE;CMAKE_CXX_FLAGS_${HEFFTE_BUILD_TYPE};CMAKE_CXX_FLAGS;MPI_CXX_COMPILER;MPI_CXX_COMPILE_OPTIONS;Heffte_ENABLE_GPU_AWARE_MPI")
 
 if (Heffte_ENABLE_CUDA)
     list(APPEND HEFFTE_OPTIONS "CMAKE_CUDA_FLAGS")

doxygen/installation.md

@@ -135,12 +135,15 @@ The `CMAKE_CUDA_COMPILER` is usually enough for a standard CUDA installation.
 
 ### GPU-Aware MPI
 
-Different implementations of MPI can provide GPU-Aware capabilities, where data can be send/received directly in GPU memory. OpenMPI provided CUDA aware capabilities if compiled with the corresponding options, e.g., see [CUDA-Aware OpenMPI](https://www.open-mpi.org/faq/?category=buildcuda). CUDA, ROCm and OneAPI support such API; however, the specific implementation available to the user may not be available for various reasons, e.g., insufficient hardware support. HeFFTe can be compiled without GPU-Aware capabilities with the CMake option:
+Different implementations of MPI can provide GPU-Aware capabilities, where data can be send/received directly in GPU memory. OpenMPI provided CUDA aware capabilities if compiled with the corresponding options, e.g., see [CUDA-Aware OpenMPI](https://www.open-mpi.org/faq/?category=buildcuda). CUDA, ROCm and OneAPI support such API; however, the specific implementation available to the user may not be available for various reasons, e.g., insufficient hardware support. HeFFTe can be compiled with and without GPU-Aware capabilities with the CMake option:
 ```
-    -D Heffte_DISABLE_GPU_AWARE_MPI=ON
+    -D Heffte_ENABLE_GPU_AWARE_MPI=<ON/OFF>
 ```
+The option is set to ON by default whenever any GPU backend has been enabled. It has no effect when using CPU only backends and is therefore set to OFF.
+
 **Note:** The GPU-Aware capabilities can also be disabled at runtime by setting the corresponding option in `heffte::plan_options`. On some platforms, the GPU-Aware MPI calls have significantly larger latency and moving the buffers to the CPU before ini
 
+**Note:** The old option `Heffte_DISABLE_GPU_AWARE_MPI` is still accepted for compatibility reasons but it is deprecated.
 
 ### Linking to HeFFTe
 

include/heffte_config.cmake.h

@@ -26,7 +26,7 @@
 
 #cmakedefine Heffte_ENABLE_TRACING
 
-#cmakedefine Heffte_DISABLE_GPU_AWARE_MPI
+#cmakedefine Heffte_ENABLE_GPU_AWARE_MPI
 
 #if defined(Heffte_ENABLE_CUDA) || defined(Heffte_ENABLE_ROCM) || defined(Heffte_ENABLE_ONEAPI)
 #define Heffte_ENABLE_GPU

src/heffte_reshape3d.cpp

@@ -22,14 +22,14 @@ namespace heffte {
     /*!
      * \internal
      * \ingroup hefftereshape
-     * \brief Struct type to convert pre-processor define Heffte_DISABLE_GPU_AWARE_MPI to boolean.
+     * \brief Struct type to convert pre-processor define Heffte_ENABLE_GPU_AWARE_MPI to boolean.
      *
      * \endinternal
      */
-#ifdef Heffte_DISABLE_GPU_AWARE_MPI
-    struct disable_gpu_aware : std::true_type {};
-#else
+#ifdef Heffte_ENABLE_GPU_AWARE_MPI
     struct disable_gpu_aware : std::false_type {};
+#else
+    struct disable_gpu_aware : std::true_type {};
 #endif