Skip to content

Commit

Permalink
Merge pull request lanl#25 from lanl/Daniel
Browse files Browse the repository at this point in the history
Looks good
  • Loading branch information
nathanielmorgan authored Nov 16, 2023
2 parents d522fb9 + e11177b commit 0de655c
Show file tree
Hide file tree
Showing 25 changed files with 997 additions and 44 deletions.
74 changes: 52 additions & 22 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ cmake_minimum_required(VERSION 3.8)
# C++17 standard required for MATAR
set(CMAKE_CXX_STANDARD 17)

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

# Build type
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE RelWithDebInfo)
Expand Down Expand Up @@ -44,14 +46,8 @@ if (WITH_DOCS)
find_package(Sphinx REQUIRED)
endif()

# MATAR submodule
# (Note: the way the MATAR submodule is handled may need to change in the
# future. Currently, it is deployed as a header-only library, which is why it
# seemed sufficient to simply include the MATAR header. However, the
# requirements of linking Kokkos and other parts of Trilinos may change this.
# Please update this as needed.)
add_subdirectory(matar)
include_directories(matar/src)
include_directories(matar/src/include)


################################################################################
Expand All @@ -73,14 +69,6 @@ add_subdirectory(slam)
add_subdirectory(swage)
add_subdirectory(io)

# Tests
if (BUILD_TESTS)
add_subdirectory(tests)
endif()

# Examples
add_subdirectory(examples)

# Documentation
if (WITH_DOCS)
add_subdirectory(docs)
Expand All @@ -98,15 +86,57 @@ add_subdirectory(mesh_tools)
# subdirectories looking for anything that matches the pattern "*.h" is
# indiscriminate, and the resulting include/ directory isn't affected by
# distclean. If there's a better way to do this, please introduce it here.)
file(GLOB_RECURSE ALL_HEADERS *.h)
add_library(Elements INTERFACE)

file(GLOB_RECURSE ALL_HEADERS elements/*.h geometry/*.h slam/*.h swage/*.h common/*.h swage2vtk/*.h)
install(FILES ${ALL_HEADERS} DESTINATION include)

target_include_directories(Elements INTERFACE
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include>
)

include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${PROJECT_BINARY_DIR}/ElementsConfigVersion.cmake"
VERSION 1.0
COMPATIBILITY AnyNewerVersion
)

install(TARGETS Elements elements geometry slam swage common matar
EXPORT ElementsTargets
LIBRARY DESTINATION lib COMPONENT Runtime
ARCHIVE DESTINATION lib COMPONENT Development
RUNTIME DESTINATION bin COMPONENT Runtime
PUBLIC_HEADER DESTINATION include COMPONENT Development
BUNDLE DESTINATION bin COMPONENT Runtime
)

include(CMakePackageConfigHelpers)
configure_package_config_file(
"${PROJECT_SOURCE_DIR}/cmake/ElementsConfig.cmake.in"
"${PROJECT_BINARY_DIR}/ElementsConfig.cmake"
INSTALL_DESTINATION lib/cmake/Elements
)

set_target_properties(Elements
PROPERTIES
PUBLIC_HEADER "${ALL_HEADERS}"
)

install(EXPORT ElementsTargets DESTINATION lib/cmake/Elements)
install(FILES "${PROJECT_BINARY_DIR}/ElementsConfigVersion.cmake"
"${PROJECT_BINARY_DIR}/ElementsConfig.cmake"
DESTINATION lib/cmake/Elements)

# Install libraries in lib/ subdirectory of the specified install directory
install(TARGETS elements DESTINATION lib)
install(TARGETS geometry DESTINATION lib)
install(TARGETS slam DESTINATION lib)
install(TARGETS swage DESTINATION lib)
if (WITH_VTK AND VTK_FOUND)
install(TARGETS swage2vtk DESTINATION lib)
install(TARGETS swage2vtk EXPORT ElementsTargets DESTINATION lib)
endif()
install(TARGETS common DESTINATION lib)

target_link_libraries(Elements INTERFACE elements geometry slam swage common)
if (WITH_VTK AND VTK_FOUND)
target_link_libraries(Elements INTERFACE swage2vtk)
endif()
target_link_libraries(Elements INTERFACE matar)

4 changes: 4 additions & 0 deletions cmake/ElementsConfig.cmake.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
@PACKAGE_INIT@

include("${CMAKE_CURRENT_LIST_DIR}/ElementsTargets.cmake")
check_required_components("@Elements@")
6 changes: 3 additions & 3 deletions common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ else()
)
endif()

target_include_directories(common PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
)
#target_include_directories(common PUBLIC
# ${CMAKE_CURRENT_SOURCE_DIR}
#)
6 changes: 3 additions & 3 deletions elements/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ target_link_libraries(
common
)

target_include_directories(elements PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
)
#target_include_directories(elements PUBLIC
# ${CMAKE_CURRENT_SOURCE_DIR}
#)
10 changes: 10 additions & 0 deletions examples/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
project(Examples)

# CMake version > 3.8 required for C++17 standard
cmake_minimum_required(VERSION 3.8)

find_package(Elements)
if(Matar_ENABLE_KOKKOS)
find_package(Kokkos)
endif()

# "Average" example, an overview of ELEMENTS
add_subdirectory(average)

Expand Down
2 changes: 2 additions & 0 deletions examples/average/src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
add_library(input input.cpp)

target_link_libraries(input Elements)
9 changes: 5 additions & 4 deletions examples/average/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ set(Average_SRC_CXX average.cpp)

add_executable(Average ${Average_SRC_CXX})

target_link_libraries (Average swage)
target_link_libraries (Average elements)
target_link_libraries (Average geometry)
target_link_libraries (Average input)
#target_link_libraries (Average swage)
#target_link_libraries (Average elements)
#target_link_libraries (Average geometry)
#target_link_libraries (Average input)
target_link_libraries (Average Elements input)
2 changes: 1 addition & 1 deletion matar
Submodule matar updated 74 files
+1 −5 .gitignore
+1 −7 .gitmodules
+47 −2 CMakeLists.txt
+19 −52 README.md
+4 −0 cmake/MatarConfig.cmake.in
+15 −8 examples/CMakeLists.txt
+3 −15 examples/halfspace_cooling/CMakeLists.txt
+7 −19 examples/laplace/CMakeLists.txt
+4 −0 examples/main.cpp
+1 −1 examples/main_kokkos.cpp
+0 −1 examples/phaseFieldMPI/heffte
+4 −4 examples/sparsetests/CMakeLists.txt
+1 −1 examples/test_rocm/CMakeLists.txt
+2 −14 examples/virtualFcnKokkos/CMakeLists.txt
+2 −1 examples/virtualFcnMATAR/CMakeLists.txt
+1 −1 examples/watt-graph/CMakeLists.txt
+25 −0 scripts/README.txt
+169 −0 scripts/build-matar.sh
+34 −0 scripts/cmake_build_examples.sh
+34 −0 scripts/cmake_build_test.sh
+0 −5 scripts/kokkos-cuda-amp/build-it.sh
+0 −19 scripts/kokkos-cuda-amp/cuda_cmake_build.sh
+0 −27 scripts/kokkos-cuda-amp/kokkos-install.sh
+0 −24 scripts/kokkos-cuda-amp/sourceme-env.sh
+0 −5 scripts/kokkos-cuda/build-it.sh
+0 −19 scripts/kokkos-cuda/cuda_cmake_build.sh
+0 −25 scripts/kokkos-cuda/kokkos-install.sh
+0 −25 scripts/kokkos-cuda/sourceme-env.sh
+0 −5 scripts/kokkos-hip/build-it.sh
+0 −21 scripts/kokkos-hip/hip_cmake_build.sh
+0 −24 scripts/kokkos-hip/kokkos-install.sh
+0 −26 scripts/kokkos-hip/sourceme-env.sh
+90 −0 scripts/kokkos-install.sh
+0 −5 scripts/kokkos-macOS-pthreads/build-it.sh
+0 −21 scripts/kokkos-macOS-pthreads/kokkos-install.sh
+0 −18 scripts/kokkos-macOS-pthreads/pthreads_cmake_build.sh
+0 −24 scripts/kokkos-macOS-pthreads/sourceme-env.sh
+0 −5 scripts/kokkos-macOS/build-it.sh
+0 −20 scripts/kokkos-macOS/kokkos-install.sh
+0 −17 scripts/kokkos-macOS/serial_cmake_build.sh
+0 −24 scripts/kokkos-macOS/sourceme-env.sh
+0 −5 scripts/kokkos-openmp/build-it.sh
+0 −21 scripts/kokkos-openmp/kokkos-install.sh
+0 −18 scripts/kokkos-openmp/openmp_cmake_build.sh
+0 −27 scripts/kokkos-openmp/sourceme-env.sh
+0 −5 scripts/kokkos-pthreads/build-it.sh
+0 −21 scripts/kokkos-pthreads/kokkos-install.sh
+0 −18 scripts/kokkos-pthreads/pthreads_cmake_build.sh
+0 −24 scripts/kokkos-pthreads/sourceme-env.sh
+0 −5 scripts/kokkos-serial/build-it.sh
+0 −20 scripts/kokkos-serial/kokkos-install.sh
+0 −17 scripts/kokkos-serial/serial_cmake_build.sh
+0 −25 scripts/kokkos-serial/sourceme-env.sh
+105 −0 scripts/legacy/cmake_build.sh
+97 −0 scripts/legacy/kokkos-install.sh
+92 −0 scripts/legacy/matar-install.sh
+68 −0 scripts/legacy/setup-env.sh
+27 −0 scripts/machines/darwin-env.sh
+12 −0 scripts/machines/linux-env.sh
+11 −0 scripts/machines/mac-env.sh
+37 −0 scripts/matar-install.sh
+0 −4 scripts/serial/build-it.sh
+0 −17 scripts/serial/cmake_build.sh
+0 −19 scripts/serial/sourceme-env.sh
+46 −0 scripts/setup-env.sh
+0 −26 src/CMakeLists.txt
+0 −0 src/Kokkos/.gitmodules
+178 −0 src/include/aliases.h
+5,219 −0 src/include/host_types.h
+8,804 −0 src/include/kokkos_types.h
+962 −0 src/include/macros.h
+87 −0 src/include/matar.h
+0 −1 test/googletest
+0 −0 test/timetest.py
25 changes: 25 additions & 0 deletions scripts/README.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
all scripts can be built fully by entering the appropriate directory and doing
source build-it.sh <arg1 arg2 ...>
build-it.sh can take up to 3 arguments (minimum of 2)
source build-it.sh <environment type> <parallelism> <build directory name (optional)>
environment has two options: 'hpc' or 'macos'
hpc: builds by loading modules and can perform parallel builds (make -j)
macos: does not load anything externally and expects the environment to be set up on your mac. Additionally, the builds will all be serial (make)
parallelism has four options: 'cuda', 'hip', 'openmp', 'none'
Note - all builds use Kokkos. The 'none' option will still use Kokkos, however only utilizing the Kokkos Serial build
cuda: loads cuda module and a working gcc module pairing (these can be changed, more info later)
hip: loads hip module and a working clang module pairing
openmp: loads gcc module and sets openmp environment variables
none: loads gcc module
build directory is an optional argument which will create the name of the build directory

All other scripts will be called with the appropriate arguments as a result of running build-it.

If you need to simply rebuild the app and not get a new kokkos installation, simply
source cmake_build.sh <args>
with the same arguments you would with build-it.sh

If you log onto a machine for the first time (or get a new allocation) you will need to run
source setup-env.sh <args>
with the same arguments you would with build-it.sh

169 changes: 169 additions & 0 deletions scripts/build-elements.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
#!/bin/bash -e
show_help() {
echo "Usage: source $(basename "$BASH_SOURCE") [OPTION]"
echo "Valid options:"
echo " --execution=<examples|test>. Default is 'all'"
echo " --kokkos_build_type=<none|serial|openmp|pthreads|cuda|hip>. Default is 'serial'"
echo " --build_action=<full-app|set-env|install-kokkos|install-elements|elements>. Default is 'full-app'"
echo " --machine=<darwin|chicoma|linux|mac>. Default is 'linux'"
echo " --build_cores=<Integers greater than 0>. Default is set 1"
echo " --help: Display this help message"
echo " "
echo " "
echo " "
echo " --build_action The desired build step to be execute. The default action is 'full-app'"
echo " "
echo " full-app builds Fierro from scratch, installing dependencies where necessary."
echo " set-env set appropriate environment variables and loads software modules (if necessary)"
echo " install-kokkos builds and installs Kokkos if not already installed. Clones from github if necessary"
echo " install-elements builds and installs Elements if not already installed."
echo " elements Generates CMake files and builds Elements only (none of the dependencies)."
echo " "
echo " --execution Builds the desired executables you want to run. The default action is 'all'"
echo " "
echo " examples builds examples"
echo " test builds tests"
echo " "
echo " --kokkos_build_type The desired kokkos parallel backend to use. The default is 'serial'"
echo " "
echo " none No Kokkos backend"
echo " serial Serial Kokkos backend"
echo " openmp OpenMP Kokkos backend"
echo " pthreads pthreads Kokkos backend"
echo " cuda Cuda Kokkos backend"
echo " hip HIP Kokkos backend"
echo " "
echo " --machine The machine you are building for. The default is 'linux'"
echo " "
echo " darwin The darwin cluster at LANL. Uses module loads for software"
echo " linux A general linux machine (that does not use modules)"
echo " mac A Mac computer. This option does not allow for cuda and hip builds, and build_cores will be set to 1"
echo " "
echo " --build_cores The number of build cores to be used by make and make install commands. The default is 1"
return 1
}

# Initialize variables with default values
build_action="full-app"
execution="examples"
machine="linux"
kokkos_build_type="serial"
build_cores="1"

# Define arrays of valid options
valid_build_action=("full-app" "set-env" "install-matar" "install-elements" "install-kokkos" "elements")
valid_execution=("examples" "test")
valid_kokkos_build_types=("none" "serial" "openmp" "pthreads" "cuda" "hip")
valid_machines=("darwin" "chicoma" "linux" "mac")

# Parse command line arguments
for arg in "$@"; do
case "$arg" in
--build_action=*)
option="${arg#*=}"
if [[ " ${valid_build_action[*]} " == *" $option "* ]]; then
build_action="$option"
else
echo "Error: Invalid --build_action specified."
show_help
return 1
fi
;;
--execution=*)
option="${arg#*=}"
if [[ " ${valid_execution[*]} " == *" $option "* ]]; then
execution="$option"
else
echo "Error: Invalid --execution specified."
show_help
return 1
fi
;;
--machine=*)
option="${arg#*=}"
if [[ " ${valid_machines[*]} " == *" $option "* ]]; then
machine="$option"
else
echo "Error: Invalid --machine specified."
show_help
return 1
fi
;;
--kokkos_build_type=*)
option="${arg#*=}"
if [[ " ${valid_kokkos_build_types[*]} " == *" $option "* ]]; then
kokkos_build_type="$option"
else
echo "Error: Invalid --kokkos_build_type specified."
show_help
return 1
fi
;;
--build_cores=*)
option="${arg#*=}"
if [ $option -ge 1 ]; then
build_cores="$option"
else
echo "Error: Invalid --build_cores specified."
show_help
return 1
fi
;;
--help)
show_help
return 1
;;
*)
echo "Error: Invalid argument or value specified."
show_help
return 1
;;
esac
done

# Check for correct combos with mac
if [ "$machine" = "mac" ] && [ "$kokkos_build_type" = "cuda" ]; then
echo "Error: Mac cannot build with Kokkos Cuda backend"
show_help
return 1
fi

if [ "$machine" = "mac" ] && [ "$kokkos_build_type" = "hip" ]; then
echo "Error: Mac cannot build with Kokkos HIP backend"
show_help
return 1
fi

if [ "$machine" = "mac" ] && [ $build_cores -ne 1 ]; then
echo "Error: Mac cannot be built in parallel. Setting build cores to default 1"
# Nothing to do, default is already 1
fi


echo "Building based on these argument options:"
echo "Build action - ${build_action}"
echo "Execution - ${execution}"
echo "Kokkos backend - ${kokkos_build_type}"
echo "make -j ${build_cores}"

cd "$( dirname "${BASH_SOURCE[0]}" )"

# Always setup the environment
source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores}

# Next, do action based on args
if [ "$build_action" = "full-app" ]; then
source kokkos-install.sh ${kokkos_build_type}
source elements-install.sh ${kokkos_build_type}
source cmake_build_${execution}.sh ${kokkos_build_type}
elif [ "$build_action" = "install-kokkos" ]; then
source kokkos-install.sh ${kokkos_build_type}
elif [ "$build_action" = "install-elements" ]; then
source elements-install.sh ${kokkos_build_type}
elif [ "$build_action" = "elements" ]; then
source cmake_build_${execution}.sh ${kokkos_build_type}
else
echo "No build action, only setup the environment."
fi

cd ${basedir}
34 changes: 34 additions & 0 deletions scripts/cmake_build_examples.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash -e

kokkos_build_type="${1}"

if [ ! -d "${EXAMPLE_SOURCE_DIR}/phaseFieldMPI/heffte" ]
then
echo "Missing heffte for examples, downloading...."
git clone https://bitbucket.org/icl/heffte.git ${EXAMPLE_SOURCE_DIR}/phaseFieldMPI/heffte
fi

cmake_options=(
-D CMAKE_PREFIX_PATH="${ELEMENTS_INSTALL_DIR};${KOKKOS_INSTALL_DIR}"
)

if [ "$kokkos_build_type" = "none" ]; then
cmake_options+=(
-D Matar_ENABLE_KOKKOS=OFF
)
else
cmake_options+=(
-D Matar_ENABLE_KOKKOS=ON
)
fi

# Print CMake options for reference
echo "CMake Options: ${cmake_options[@]}"

# Configure Examples
cmake "${cmake_options[@]}" -B "${EXAMPLE_BUILD_DIR}" -S "${EXAMPLE_SOURCE_DIR}"

# Build Examples
make -C "${EXAMPLE_BUILD_DIR}" -j${ELEMENTS_BUILD_CORES}

cd $basedir
Loading

0 comments on commit 0de655c

Please sign in to comment.