From e950c28b724de7cea04f39da38b4d7a37ad1395f Mon Sep 17 00:00:00 2001 From: Miroslav Stoyanov Date: Wed, 23 Oct 2024 17:30:06 -0400 Subject: [PATCH] improved the gpu-aware option --- CMakeLists.txt | 10 ++++++++++ cmake/HeffteConfig.cmake | 6 +++--- cmake/print_summary.cmake | 2 +- doxygen/installation.md | 7 +++++-- include/heffte_config.cmake.h | 2 +- src/heffte_reshape3d.cpp | 8 ++++---- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe7e198..41e36c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,16 @@ option(Heffte_ENABLE_TRACING "Enable the tracing capabilities" OFF) option(BUILD_SHARED_LIBS "Builds shared libraries using CMake conventions" ON) +if (Heffte_ENABLE_CUDA OR Heffte_ENABLE_ROCM OR Heffte_ENABLE_ONEAPI) + if (Heffte_DISABLE_GPU_AWARE_MPI) + option(Heffte_ENABLE_GPU_AWARE_MPI "GPU to GPU direct MPI calls" OFF) + else() + option(Heffte_ENABLE_GPU_AWARE_MPI "GPU to GPU direct MPI calls" ON) + endif() +else() + set(Heffte_ENABLE_GPU_AWARE_MPI OFF CACHE BOOL "GPU to GPU direct MPI calls" FORCE) +endif() + if (${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME}) option(Heffte_ENABLE_TESTING "Enable the internal unit tests" ON) else() diff --git a/cmake/HeffteConfig.cmake b/cmake/HeffteConfig.cmake index 1c0e78f..94f6067 100644 --- a/cmake/HeffteConfig.cmake +++ b/cmake/HeffteConfig.cmake @@ -107,10 +107,10 @@ set(Heffte_CUDA_FOUND "@Heffte_ENABLE_CUDA@") set(Heffte_ROCM_FOUND "@Heffte_ENABLE_ROCM@") set(Heffte_ONEAPI_FOUND "@Heffte_ENABLE_ONEAPI@") set(Heffte_INTRINSICS_FOUND "@Heffte_ENABLE_INTRINSICS@") -if ("@Heffte_DISABLE_GPU_AWARE_MPI@") - set(Heffte_GPUAWARE_FOUND "OFF") -else() +if ("@Heffte_ENABLE_GPU_AWARE_MPI@") set(Heffte_GPUAWARE_FOUND "ON") +else() + set(Heffte_GPUAWARE_FOUND "OFF") endif() check_required_components(Heffte) diff --git a/cmake/print_summary.cmake b/cmake/print_summary.cmake index 6598c5e..1b20c09 100644 --- a/cmake/print_summary.cmake +++ b/cmake/print_summary.cmake @@ -3,7 +3,7 @@ message(STATUS "") message(STATUS "heFFTe ${PROJECT_VERSION}") string(TOUPPER "${CMAKE_BUILD_TYPE}" HEFFTE_BUILD_TYPE) -set(HEFFTE_OPTIONS "CMAKE_INSTALL_PREFIX;BUILD_SHARED_LIBS;CMAKE_BUILD_TYPE;CMAKE_CXX_FLAGS_${HEFFTE_BUILD_TYPE};CMAKE_CXX_FLAGS;MPI_CXX_COMPILER;MPI_CXX_COMPILE_OPTIONS") +set(HEFFTE_OPTIONS "CMAKE_INSTALL_PREFIX;BUILD_SHARED_LIBS;CMAKE_BUILD_TYPE;CMAKE_CXX_FLAGS_${HEFFTE_BUILD_TYPE};CMAKE_CXX_FLAGS;MPI_CXX_COMPILER;MPI_CXX_COMPILE_OPTIONS;Heffte_ENABLE_GPU_AWARE_MPI") if (Heffte_ENABLE_CUDA) list(APPEND HEFFTE_OPTIONS "CMAKE_CUDA_FLAGS") diff --git a/doxygen/installation.md b/doxygen/installation.md index c7d7934..e3a73c5 100644 --- a/doxygen/installation.md +++ b/doxygen/installation.md @@ -135,12 +135,15 @@ The `CMAKE_CUDA_COMPILER` is usually enough for a standard CUDA installation. ### GPU-Aware MPI -Different implementations of MPI can provide GPU-Aware capabilities, where data can be send/received directly in GPU memory. OpenMPI provided CUDA aware capabilities if compiled with the corresponding options, e.g., see [CUDA-Aware OpenMPI](https://www.open-mpi.org/faq/?category=buildcuda). CUDA, ROCm and OneAPI support such API; however, the specific implementation available to the user may not be available for various reasons, e.g., insufficient hardware support. HeFFTe can be compiled without GPU-Aware capabilities with the CMake option: +Different implementations of MPI can provide GPU-Aware capabilities, where data can be send/received directly in GPU memory. OpenMPI provided CUDA aware capabilities if compiled with the corresponding options, e.g., see [CUDA-Aware OpenMPI](https://www.open-mpi.org/faq/?category=buildcuda). CUDA, ROCm and OneAPI support such API; however, the specific implementation available to the user may not be available for various reasons, e.g., insufficient hardware support. HeFFTe can be compiled with and without GPU-Aware capabilities with the CMake option: ``` - -D Heffte_DISABLE_GPU_AWARE_MPI=ON + -D Heffte_ENABLE_GPU_AWARE_MPI= ``` +The option is set to ON by default whenever any GPU backend has been enabled. It has no effect when using CPU only backends and is therefore set to OFF. + **Note:** The GPU-Aware capabilities can also be disabled at runtime by setting the corresponding option in `heffte::plan_options`. On some platforms, the GPU-Aware MPI calls have significantly larger latency and moving the buffers to the CPU before ini +**Note:** The old option `Heffte_DISABLE_GPU_AWARE_MPI` is still accepted for compatibility reasons but it is deprecated. ### Linking to HeFFTe diff --git a/include/heffte_config.cmake.h b/include/heffte_config.cmake.h index f5d094b..ac2c30c 100644 --- a/include/heffte_config.cmake.h +++ b/include/heffte_config.cmake.h @@ -26,7 +26,7 @@ #cmakedefine Heffte_ENABLE_TRACING -#cmakedefine Heffte_DISABLE_GPU_AWARE_MPI +#cmakedefine Heffte_ENABLE_GPU_AWARE_MPI #if defined(Heffte_ENABLE_CUDA) || defined(Heffte_ENABLE_ROCM) || defined(Heffte_ENABLE_ONEAPI) #define Heffte_ENABLE_GPU diff --git a/src/heffte_reshape3d.cpp b/src/heffte_reshape3d.cpp index 48099c2..5f39b89 100644 --- a/src/heffte_reshape3d.cpp +++ b/src/heffte_reshape3d.cpp @@ -22,14 +22,14 @@ namespace heffte { /*! * \internal * \ingroup hefftereshape - * \brief Struct type to convert pre-processor define Heffte_DISABLE_GPU_AWARE_MPI to boolean. + * \brief Struct type to convert pre-processor define Heffte_ENABLE_GPU_AWARE_MPI to boolean. * * \endinternal */ -#ifdef Heffte_DISABLE_GPU_AWARE_MPI - struct disable_gpu_aware : std::true_type {}; -#else +#ifdef Heffte_ENABLE_GPU_AWARE_MPI struct disable_gpu_aware : std::false_type {}; +#else + struct disable_gpu_aware : std::true_type {}; #endif