added compilation mn5 suiteSparse and jsonfortran

compile/power9/check.sh → compile/check.sh

@@ -1,5 +1,5 @@
 set -e
-cd ../../build
+cd ../build
 make -j 4
 ctest --output-on-failure
 #./unit_test_aero_rep_single_particle

compile/power9/compile.camp.sh → compile/compile.camp.sh

@@ -1,10 +1,27 @@
 #!/usr/bin/env bash
 
-relative_path="../../../"
+#Run script from anywhere
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+cd $DIR
+
+relative_path="../../"
 curr_path=$(pwd)
 
-LOCAL_MACHINE=CGUZMAN
-if [ $BSC_MACHINE == "power" ]; then
+if [ $BSC_MACHINE == "mn5" ]; then
+  module load cmake
+  module load gcc
+  module load openmpi/4.1.5-gcc
+  module load mkl
+  module load hdf5/1.14.1-2-gcc-ompi
+  module load python/3.12.1-gcc
+  module load netcdf/c-4.9.2_fortran-4.6.1_cxx4-4.3.1-gcc-ompi
+elif [ $BSC_MACHINE == "power" ]; then
   module load GCC/7.3.0-2.30
   module load OpenMPI/3.1.0-GCC-7.3.0-2.30
   module load JasPer/1.900.1-foss-2018b
@@ -31,16 +48,12 @@ elif [ $BSC_MACHINE == "mn4" ]; then
   module load netcdf/4.4.1.1
   module load hdf5/1.8.19
   module load libpng/1.5.13
-elif [ LOCAL_MACHINE==CGUZMAN ]; then
+else
   mpifort=$(which mpifort)
   if ! command -v mpicc &> /dev/null; then
       echo "MPI is not installed. Installing..."
       sudo apt update
       sudo apt install -y mpi-default-dev
-  fi
-else
-  echo "Unknown architecture"
-  exit
 fi
 export SUNDIALS_HOME=$(pwd)/$relative_path/cvode-3.4-alpha/install
 export SUITE_SPARSE_HOME=$(pwd)/$relative_path/SuiteSparse

compile/power9/compile.cvode-3.4-alpha.power9.sh → compile/compile.cvode-3.4-alpha.power9.sh

@@ -1,8 +1,25 @@
 #!/usr/bin/env bash
 
-library_path="../../../"
+library_path="../../"
 curr_path=$(pwd)
 
+if [ $BSC_MACHINE == "mn5" ]; then
+module load cmake
+module load gcc
+module load Ppenmpi
+  module load GCC/7.3.0-2.30
+  module load OpenMPI/3.1.0-GCC-7.3.0-2.30
+  module load JasPer/1.900.1-foss-2018b
+  module load netCDF/4.6.1-foss-2018b
+  module load netCDF-Fortran/4.4.4-foss-2018b
+  module load ESMF/6.3.0rp1-foss-2018b
+  module load CMake/3.15.3-GCCcore-7.3.0
+  module load OpenBLAS/0.3.1-GCC-7.3.0-2.30
+  module load CUDA/10.1.105-ES
+  module load Python/3.7.0-foss-2018b
+  module load matplotlib/3.1.1-foss-2018b-Python-3.7.0
+fi
+
 if [ -z "$SUITE_SPARSE_CAMP_ROOT" ]; then
 	SUITE_SPARSE_CAMP_ROOT=$(pwd)/$library_path/SuiteSparse
 fi

compile/power9/compile.json-fortran-6.1.0.power9.sh → compile/compile.json-fortran-6.1.0.power9.sh

@@ -1,11 +1,19 @@
 #!/usr/bin/env bash
+set -e
 
-library_path="../../../"
+if [ $BSC_MACHINE == "mn5" ]; then
+module load gcc
+module load openmpi/4.1.5-gcc
+module load cmake
+#module load intel
+fi
+
+library_path="../../"
 curr_path=$(pwd)
 cd $library_path/json-fortran-6.1.0
-rm -r build
+rm -rf build
 mkdir build
-mkdir install
+mkdir install || true
 cd build
 cmake -D SKIP_DOC_GEN:BOOL=TRUE -D CMAKE_INSTALL_PREFIX=$(pwd)/../install ..
 make install

compile/compile.libs.camp.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+./compile.json-fortran-6.1.0.power9.sh
+./compile.suiteSparse.power9.sh
+./compile.cvode-3.4-alpha.power9.sh
+./compile.camp.sh
+

compile/compile.suiteSparse.power9.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -e
+library_path="../.."
+curr_path=$(pwd)
+
+cd $library_path/OpenBLAS
+path_Blas_install=$(pwd)/install/
+compile_BLAS(){
+make
+mkdir install || true
+make install PREFIX=$path_Blas_install
+}
+cd $curr_path
+#compile_BLAS
+
+cd $library_path/SuiteSparse
+make purge
+if [ $BSC_MACHINE == "mn5" ] || [$BSC_MACHINE == "power"] ; then
+  module load cmake
+  module load gcc
+  module load openmpi/4.1.5-gcc
+  make BLAS="-L/usr/lib/x86_64-linux-gnu -I$path_Blas_install/include/ -L$path_Blas_install/lib -Wl,-rpath,$path_Blas_install/OpenBLAS/lib -lopenblas" LAPACK=""
+  #module load intel
+  #module load openmpi
+  #make BLAS="-L${INTEL_HOME}/mkl/lib/intel64 -lpthread -lm" LAPACK="" #wrong -openmp and missing BLAS
+elif [ $BSC_MACHINE == "mn4" ]; then
+  make BLAS="-L${INTEL_HOME}/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lpthread -lm" LAPACK=""
+else
+  make BLAS="-L/usr/lib/x86_64-linux-gnu -lopenblas" LAPACK=""
+fi
+export SUITE_SPARSE_CAMP_ROOT=$(pwd)/$library_path/
+cd $curr_path

compile/copy_bashrc

@@ -0,0 +1,211 @@
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# If not running interactively (non bash console gui), don't do anything. Used for exit bashrc when using autosubmit, avoiding to load unneeded modules
+[ -z "$PS1" ] && return
+
+# Manual alias
+load_alias(){
+alias bsc32815="cd /gpfs/scratch/bsc32/bsc32815/"
+alias a1vg="cd /gpfs/scratch/bsc32/bsc32815/a1vg/nmmb-monarch/MODEL/SRC_LIBS"
+alias a209="cd /gpfs/scratch/bsc32/bsc32815/a209/nmmb-monarch/MODEL/SRC_LIBS"
+alias a2ry="cd /gpfs/scratch/bsc32/bsc32815/a2ry/nmmb-monarch"
+alias a2s8="cd /gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch"
+alias a2sv="cd /gpfs/scratch/bsc32/bsc32815/a2sv/nmmb-monarch/MODEL/SRC_LIBS"
+alias a582="cd /gpfs/scratch/bsc32/bsc32815/a582/nmmb-monarch"
+alias a582c="cd /gpfs/scratch/bsc32/bsc32815/a582/nmmb-monarch/MODEL/SRC_LIBS/camp/compile/power9"
+alias a5lc="cd /gpfs/scratch/bsc32/bsc32815/a5lc/nmmb-monarch"
+alias a5lcc="cd /gpfs/scratch/bsc32/bsc32815/a5lc/nmmb-monarch/MODEL/SRC_LIBS/camp/compile/power9"
+alias a5lco="cd /gpfs/scratch/bsc32/bsc32815/a5lc/nmmb-monarch/OUTPUT/regional/000/20160721"
+alias a5bj="cd /gpfs/scratch/bsc32/bsc32815/a5bj/nmmb-monarch"
+alias a5bjc="cd /gpfs/scratch/bsc32/bsc32815/a5bj/nmmb-monarch/MODEL/SRC_LIBS/camp/compile/power9"
+alias a5bjo="cd /gpfs/scratch/bsc32/bsc32815/a5bj/nmmb-monarch/OUTPUT/regional/000/20160721"
+alias a591="cd /gpfs/scratch/bsc32/bsc32815/a591/nmmb-monarch"
+alias a591c="cd /gpfs/scratch/bsc32/bsc32815/a591/nmmb-monarch/MODEL/SRC_LIBS/camp/compile/power9"
+alias a591t="cd /gpfs/scratch/bsc32/bsc32815/a591/nmmb-monarch/MODEL/SRC_LIBS/camp/test/monarch"
+alias a591l="cd /gpfs/scratch/bsc32/bsc32815/a591/LOG_a591"
+alias a591o="cd /gpfs/scratch/bsc32/bsc32815/a591/nmmb-monarch/OUTPUT/regional/000/20160721"
+alias a591mo="cd /gpfs/scratch/bsc32/bsc32815/a591/nmmb-monarch/MODEL/SRC_LIBS/camp/test/monarch_output"
+alias a5hl="cd /gpfs/scratch/bsc32/bsc32815/a5hl/nmmb-monarch"
+alias a5hlo="cd /gpfs/scratch/bsc32/bsc32815/a5hl/nmmb-monarch/OUTPUT/regional/000/20160721"
+alias a5hlc="cd /gpfs/scratch/bsc32/bsc32815/a5hl/nmmb-monarch/MODEL/SRC_LIBS/camp/compile/power9"
+alias gpupartmc="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp/compile/power9"
+alias out1="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp/test/monarch"
+alias out2="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp2/test/monarch"
+alias out3="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp3/test/monarch"
+alias gpupartmcp="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp/test/monarch"
+alias gpupartmc2="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp2/compile/power9"
+alias gpupartmc3="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/camp3/compile/power9"
+alias reduce="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/reduce/wrapperCCUDA"
+alias openmp="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/MultiGPUOpenMPCAMP"
+alias ebic="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/ebic"
+alias pumps="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/pumps_cuda_22"
+alias loki2="cd /gpfs/scratch/bsc32/bsc32815/gpupartmc/LOKI/loki-example"
+}
+load_alias
+
+#module load ddt
+#ddt &
+
+power_esarchive="yes" #monarch integration
+#power_esarchive="no" #gpupartmc clion tests camp solo run
+
+if [ $BSC_MACHINE == "power" ]; then
+
+#gpupartmc
+#a591c
+
+if [ $HOSTNAME == "p9login2" ]; then #Execute only in plogin2
+
+if [ -z ${SLURM_TASK_PID+x} ]; then #Execute only if is not set already
+
+alias bsc32815="cd /gpfs/scratch/bsc32/bsc32815/"
+#salloc --x11 -t 08:00:00 --ntasks-per-node=40 --nodes=1 --gres=gpu:1
+#salloc --x11 -t 08:00:00 --ntasks-per-node=80 --nodes=1 --gres=gpu:2
+#salloc --x11 --qos=debug --tasks-per-node=80 --nodes=1 --gres=gpu:2 #monarch out of memory
+
+salloc --x11 --qos=debug --exclusive --tasks-per-node=160 --nodes=1 --gres=gpu:4
+
+#salloc --x11 --exclusive -t 02:00:00 --tasks-per-node=160 --nodes=2 --gres=gpu:4
+#WRONG RESULTS MONARCH GPU:salloc --x11 --qos=debug --exclusive --nodes=1 --ntasks-per-node=40 --cpus-per-task=4 --gres=gpu:4
+
+#40 1
+#--exclusive --qos=debug
+fi
+fi
+
+if [ $power_esarchive == "yes" ]; then
+
+#echo "power_esarchive start"
+
+#  commented: default cte-power modules
+#  module unuse /gpfs/projects/bsc32/software/rhel/7.4/ppc64le/POWER9/modules/all/
+#  module use /apps/modules/modulefiles/applications
+
+  module purge
+  #module unuse /apps/modules/modulefiles/applications #todo check if needed
+  module use /gpfs/projects/bsc32/software/rhel/7.4/ppc64le/POWER9/modules/all/
+  #module purge #todo check if needed
+  module load bsc/commands
+  module load CUDA/10.1.105-ES
+  #Adding pgcc:
+  #module load pgi
+  #module load cmake
+  #pgcc
+
+#MONARCH P9 compilation
+
+  module load GCC/7.3.0-2.30
+  module load OpenMPI/3.1.0-GCC-7.3.0-2.30
+  module load JasPer/1.900.1-foss-2018b
+  module load netCDF/4.6.1-foss-2018b
+  module load netCDF-Fortran/4.4.4-foss-2018b
+  module load ESMF/6.3.0rp1-foss-2018b
+  module load ESMF/6.3.0rp1-foss-2018b
+  module load CMake/3.15.3-GCCcore-7.3.0
+  module load OpenBLAS/0.3.1-GCC-7.3.0-2.30
+  module load Python/3.7.0-foss-2018b
+  module load matplotlib/3.1.1-foss-2018b-Python-3.7.0
+  module load netcdf4-python/1.5.1.2-foss-2018b-Python-3.7.0
+  #module load GSL/2.4-GCCcore-7.3.0
+  export ALLINEA_FORCE_CUDA_VERSION=9.2 #needed for ddt cuda
+  #module load ddt
+
+load_ncview(){
+  module load netcdf
+  module load ncview
+}
+#load_ncview
+
+#todo: needed?
+export NETCDF_FORTRAN_HOME=${EBROOTNETCDFMINFORTRAN}
+export NETCDF_HOME=${EBROOTNETCDF}
+export NETCDF_FORTRAN_LIB="/gpfs/projects/bsc32/software/rhel/7.5/ppc64le/POWER9/software/netCDF-Fortran/4.4.4-foss-2018b/lib/libnetcdff.so"
+export NETCDF_INCLUDE_DIR="/gpfs/projects/bsc32/software/rhel/7.5/ppc64le/POWER9/software/netCDF/4.6.1-foss-2018b/include"
+
+else
+
+  #Configuration for Clion gpupartmc without esarchive modules:
+  module load gcc/6.4.0
+  module load openmpi/3.0.0
+  module load hdf4/4.2.13
+  module load hdf5/1.8.20
+  module load pnetcdf/1.9.0
+  module load netcdf/4.4.1.1
+  module load lapack/3.8.0
+  module load cuda/9.1
+  module load cmake/3.11.1
+  module load hdf4/4.2.13 hdf5/1.8.20 pnetcdf/1.9.0
+  module load llvm
+  module load clang
+  module load python
+
+fi
+
+else
+#mn4
+
+if [ $HOSTNAME == "login3" ]; then #Execute only in plogin2
+if [ -z ${SLURM_TASK_PID+x} ]; then #Execute only if is not set already
+
+alias bsc32815="cd /gpfs/scratch/bsc32/bsc32815/"
+#wrong mpi run salloc --x11 --qos=debug --exclusive --ntasks-per-node=48 --nodes=1
+salloc --x11 --qos=debug --cpus-per-task=48 --ntasks=1
+#wrong salloc --x11 --qos=debug --cpus-per-task=24 --ntasks=1
+
+#40 1
+#--x11 --exclusive
+fi
+fi
+
+#Load MONARCH and CAMP modules
+load_camp_mn4(){ 
+  module load netcdf/4.4.1.1
+  module load gsl
+   module load hdf5/1.8.19
+   module load python
+}
+load_camp_mn4
+
+load_ncview(){
+  #module load netcdf
+  module load udunits
+  #module load gsl
+  module load nco
+  module load ncview
+}
+#load_ncview
+
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/wgrib-1.8.1.2
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/cvode-3.4-alpha/install
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/SuiteSparse
+# export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/json-fortran-6.1.0/install/jsonfortran-intel-6.1.0/lib
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/json-fortran-6.1.0/install/jsonfortran-intel-6.1.0/lib/
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/SuiteSparse/lib/
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/scratch/bsc32/bsc32815/a2s8/nmmb-monarch/MODEL/SRC_LIBS/cvode-3.4-alpha/install/lib/
+
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/wgrib-1.8.1.2
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/cvode-3.4-alpha/install
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/SuiteSparse
+#export PATH=$PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/json-fortran-6.1.0/install/jsonfortran-intel-6.1.0/lib
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/json-fortran-6.1.0/install/jsonfortran-intel-6.1.0/lib/
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/SuiteSparse/lib/
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/scratch/bsc32/bsc32815/a2s6/nmmb-monarch/MODEL/SRC_LIBS/cvode-3.4-alpha/install/lib/
+
+fi
+
+if [ -f $HOME/.last_folder_config ]; then
+    last_folder=$(cat $HOME/.last_folder_config)
+    if [ -d "$last_folder" ]; then
+        cd "$last_folder"
+    fi
+fi
+
+# Add a function to the end of .bashrc that saves the current folder on exit
+function save_last_folder() {
+    echo "$PWD" > $HOME/.last_folder_config
+}
+trap save_last_folder EXIT
+

compile/cuda_memcheck.sh

@@ -0,0 +1,24 @@
+compile(){
+   export SUNDIALS_HOME=$(pwd)/../../cvode-3.4-alpha/install
+   export SUITE_SPARSE_HOME=$(pwd)/../../SuiteSparse
+   export JSON_FORTRAN_HOME=$(pwd)/../../json-fortran-6.1.0/install/jsonfortran-gnu-6.1.0
+
+   if [ $BSC_MACHINE == "power" ]; then
+     export JSON_FORTRAN_HOME=$(pwd)/../../json-fortran-6.1.0/install/jsonfortran-gnu-6.1.0
+   elif [ $BSC_MACHINE == "mn4" ]; then
+     export JSON_FORTRAN_HOME=$(pwd)/../../json-fortran-6.1.0/install/jsonfortran-intel-6.1.0
+   else
+     echo "Unknown architecture"
+     exit
+   fi
+
+   curr_path=$(pwd)
+   cd  ../build
+   if ! make -j ${NUMPROC}; then
+     exit
+   fi
+   cd $curr_path
+}
+time compile
+cd ../build
+cuda-memcheck mock_monarch 2>&1 | tee "../compile/a.txt"