ARC4 Compilation modifications and instructions (Leeds-MONC#28)

* Included source code issues from issue 11, point 1
* Included changes from trunk to allow build with gnu>=7. Created arc
compatible batch script. Added diagnostic file assignment to straka.mcf
* Added stdout folder and sge outputs to .gitignore
* Included MVAPICH2 env variables in job submission script - needed.

.gitignore

@@ -5,7 +5,9 @@ fcm-make-as-parsed.cfg
 fcm-make-on-success.cfg
 fcm-make.log
 preprocess/
+monc_stdout/
 
+submonc.sge.[oe]*
 .*.sw*
 *.nc
 *.autogen

README.md

@@ -20,6 +20,16 @@ complete re-write of the Met Office LEM using modern software design with
 a flexible plug 'n play component based architecture with a focus on high
 performance computing (HPC) scaling and efficiency.  '
 
+# Compiling and Running on Leeds ARC machines
+
+To make compilation more simple on the ARC systems, a script called `monc_compile_arc.sh` has been written which loads the necessary modules and compiles MONC using fcm. A second script, `submonc.sge`, can be used to submit a MONC task using the command `qsub submonc.sge` and similarly loads the required modules before submitting the job.
+
+### MPI libraries and compilers on ARC
+
+Testing has shown MONC to be incompatible with intel compilers at present (at the very least with intel 19.04), and so the compilation script uses the gnu compiler (version 4.8.5 by default on arc4, but tested and working with gcc 8.3.0 and gcc 6.3.0). 
+
+MONC has been known to sometimes have issues with openmpi, and so mvapich2 is recommended. Mvapich2 is incapable of running with multiple threads however, and testing has shown successful runs using all three MPI variants available on arc4 (openmpi, intelmpi and mvapich2) so while mvapich2 is the recommended variant, its use remains only a recommendation rather than a necessity.
+
 # Contributing
 
 > **Note**: Github have written their own guides on [what git

components/conditional_diagnostics_whole/src/conditional_diagnostics_whole.F90

@@ -10,7 +10,7 @@ module conditional_diagnostics_whole_mod
   use conditional_diagnostics_column_mod, only : CondDiags_tot, ncond, ndiag, gpts_total, requested_area
   use grids_mod, only : Z_INDEX
   use datadefn_mod, only : PRECISION_TYPE, DEFAULT_PRECISION
-  use mpi, only : MPI_SUM, MPI_IN_PLACE, MPI_INT, MPI_REAL, MPI_DOUBLE, MPI_Comm
+  use mpi, only : MPI_SUM, MPI_IN_PLACE, MPI_INT, MPI_REAL, MPI_DOUBLE
   use missing_data_mod, only: rmdi
   use optionsdatabase_mod, only : options_get_integer
 

components/pdf_analysis/src/pdf_analysis.F90

@@ -7,7 +7,7 @@ module pdf_analysis_mod
   use grids_mod, only : X_INDEX, Y_INDEX, Z_INDEX
   use datadefn_mod, only : DEFAULT_PRECISION, PRECISION_TYPE
   use optionsdatabase_mod, only : options_has_key, options_get_logical, options_get_integer, options_get_string, options_get_real
-  use mpi, only : MPI_SUM, MPI_IN_PLACE, MPI_INT, MPI_REAL, MPI_DOUBLE, MPI_Comm
+  use mpi, only : MPI_SUM, MPI_IN_PLACE, MPI_INT, MPI_REAL, MPI_DOUBLE
   use logging_mod, only : LOG_INFO, LOG_DEBUG, LOG_ERROR, log_master_log, log_is_master
   use conversions_mod, only : conv_to_string
   implicit none

fcm-make/comp-intel-19.04.cfg

@@ -0,0 +1,15 @@
+#$f90{?} = mpif90
+#$cc{?} = cc
+$fflags_opt{?}   = -O3 -module .
+$fflags{?}       = $fflags_opt
+
+$fflags_core{?} = $fflags -recursive
+$fflags_io{?} = $fflags -recursive
+$fflags_psrc{?}  = -O1 -module .
+$cflags = 
+
+$fppdefs_debug{?} =  DEBUG_MODE
+$fpp_flags{?} = -E -P -traditional
+
+$fppdefs{?} = U_ACTIVE V_ACTIVE W_ACTIVE ENFORCE_THREAD_SAFETY __DARWIN
+$cppdefs{?} = __DARWIN _XOPEN_SOURCE=700 ENFORCE_THREAD_SAFETY

fcm-make/env-arc4.cfg

@@ -0,0 +1,13 @@
+$f90{?} = mpif90
+$cc{?} = cc
+
+$netcdf_path{?}     = $NETCDF_HOME
+$netcdf_inc_path{?} = $netcdf_path/include
+$netcdf_lib_path{?} = $netcdf_path/lib
+
+$fftw_path{?}       = $FFTW_HOME
+$fftw_inc_path{?}   = $fftw_path/include
+$fftw_lib_path{?}   = $fftw_path/lib
+
+$hdf5_path{?}       = $HDF5_HOME
+$hdf5_lib_path{?}   = $hdf5_path/lib

fcm-make/monc-arc4-gnu.cfg

@@ -0,0 +1,3 @@
+include = comp-gnu-4.4.7.cfg
+include = env-arc4.cfg
+include = monc-build.cfg

io/src/mpicommunication.F90

@@ -255,7 +255,7 @@ logical function test_for_inter_io(inter_io_communications, number_of_inter_io,
       if (message_pending) then
         call mpi_get_count(status, MPI_BYTE, message_size, ierr)
         allocate(data_buffer(message_size))
-        call mpi_recv(data_buffer, message_size, MPI_BYTE, MPI_ANY_SOURCE, inter_io_communications(i)%message_tag, &
+        call mpi_recv(data_buffer, message_size, MPI_BYTE, status(MPI_SOURCE), inter_io_communications(i)%message_tag, &
              io_communicator, MPI_STATUS_IGNORE, ierr)
         call unlock_mpi()
         command=INTER_IO_COMMUNICATION

model_core/src/components/monc_component.F90

@@ -33,6 +33,15 @@ module monc_component_mod
        logical :: enabled
     end type component_field_information_type
 
+    type, public :: pointer_wrapper_value_type
+      procedure(component_get_field_value), nopass, pointer :: ptr
+    end type pointer_wrapper_value_type
+
+    type, public :: pointer_wrapper_info_type
+      procedure(component_get_field_information), nopass, pointer :: ptr
+    end type pointer_wrapper_info_type
+
+
     !> Description of a component
     !!
     !! Provided to the registry to register a component. It is also used by the registry to store information

model_core/src/components/registry.F90

@@ -8,7 +8,8 @@ module registry_mod
        c_get_string, c_get_generic, c_remove, c_put_generic, c_put_string, c_put_integer, c_put_real, c_is_empty, &
        c_contains, c_add_generic, c_add_string, c_free, c_get_iterator, c_has_next, c_next_mapentry
   use monc_component_mod, only : component_descriptor_type, component_field_value_type, component_field_information_type, &
-       FINALISATION_PRIORITY_INDEX, INIT_PRIORITY_INDEX, TIMESTEP_PRIORITY_INDEX
+       FINALISATION_PRIORITY_INDEX, INIT_PRIORITY_INDEX, TIMESTEP_PRIORITY_INDEX, &
+       pointer_wrapper_value_type, pointer_wrapper_info_type
   use conversions_mod, only : conv_to_string
   use state_mod, only : model_state_type
   use optionsdatabase_mod, only : options_has_key, options_get_string, options_get_logical, options_get_array_size
@@ -150,7 +151,7 @@ function get_component_field_value(current_state, name)
     if (c_contains(field_procedure_retrievals, name)) then
       data=>c_get_generic(field_procedure_retrievals, name)
       select type(data)
-      type is (pointer_wrapper_type)
+      type is (pointer_wrapper_value_type)
         call data%ptr(current_state, name, get_component_field_value)
       end select
     else
@@ -171,7 +172,7 @@ function get_component_field_information(current_state, name)
     if (c_contains(field_procedure_retrievals, name)) then
       data=>c_get_generic(field_procedure_sizings, name)
       select type(data)
-      type is (pointer_wrapper_type)
+      type is (pointer_wrapper_info_type)
         call data%ptr(current_state, name, get_component_field_information)
       end select
     else
@@ -194,21 +195,23 @@ subroutine load_published_fields(descriptor)
 
     integer :: i
     class(*), pointer :: field_generic_description
-    type(pointer_wrapper_type), pointer :: wrapper
+    type(pointer_wrapper_info_type), pointer :: wrapper_info
+    type(pointer_wrapper_value_type), pointer :: wrapper_value
     class(*), pointer :: genericwrapper
 
     if (associated(descriptor%published_fields) .and. associated(descriptor%field_value_retrieval) .and. &
          associated(descriptor%field_information_retrieval)) then
       do i=1, size(descriptor%published_fields)
         field_generic_description=>descriptor%published_fields(i)
         call c_add_generic(field_information, field_generic_description, .false.)
-        allocate(wrapper) ! We allocate our own copy of the descriptor here to ensure the consistency of registry information
-        wrapper%ptr => descriptor%field_value_retrieval
-        genericwrapper=>wrapper
+        allocate(wrapper_value) ! We allocate our own copy of the descriptor here to ensure the consistency of registry information
+        wrapper_value%ptr => descriptor%field_value_retrieval
+        genericwrapper=>wrapper_value
         call c_put_generic(field_procedure_retrievals, descriptor%published_fields(i), genericwrapper, .false.)
-        allocate(wrapper)
-        wrapper%ptr => descriptor%field_information_retrieval
-        genericwrapper=>wrapper
+
+        allocate(wrapper_info)
+        wrapper_info%ptr => descriptor%field_information_retrieval
+        genericwrapper=>wrapper_info
         call c_put_generic(field_procedure_sizings, descriptor%published_fields(i), genericwrapper, .false.)
       end do
 

submonc_no_crun.pbs

@@ -26,7 +26,7 @@ export PBS_O_WORKDIR=$(readlink -f $PBS_O_WORKDIR)
 cd $PBS_O_WORKDIR
 
 # Set the number of threads to 1
-#   This prevents any system libraries from automatically  
+#   This prevents any system libraries from automatically
 #   using threading.
 export OMP_NUM_THREADS=1
 

testcases/tank_experiments/straka.mcf

@@ -15,7 +15,7 @@ diffusion_enabled=.true.
 simplecloud_enabled=.false.
 coriolis_enabled=.false.
 damping_enabled=.false.
-forcing_enabled=.false. 
+forcing_enabled=.false.
 galilean_transformation=.false. # Needs debugging
 randomnoise_enabled=.false.
 mean_profiles_enabled=.true. #This must be set to true if running with damping
@@ -34,6 +34,7 @@ dynamics_group_contents=kidtestcase, pw_advection, tvd_advection, th_advection,
 
 # IO server configuration
 ioserver_configuration_file="io/io_cfg_files/data_write_1file.xml"
+diagnostic_file="diagnostic_files/straka.nc"
 moncs_per_io_server=3
 sampling_frequency=4
 3d_sampling_frequency=20
@@ -104,7 +105,7 @@ f_init_pl_theta=300., 300.
 l_init_pl_u=.false.
 l_init_pl_v=.false.
 l_init_pl_q=.false.
-  
+
 l_matchthref=.false.
 
 # Set up the tank

utils/arc/monc_compile_arc.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+#Script to compile Monc on ARC4:
+
+module purge
+module load user
+module switch intel gnu/8.3.0
+#module switch openmpi mvapich2
+module load fftw netcdf hdf5 fcm
+
+fcm make -j4 -f fcm-make/monc-arc4-gnu.cfg

utils/arc/submonc.sge

@@ -0,0 +1,47 @@
+#!/bin/bash
+#$ -pe ib 4 
+#$ -l h_rt=0:30:0
+#$ -l h_vmem=1G
+#$ -l placement=scatter
+#$ -cwd -V
+
+##############################################################
+#### THIS IS AN EXAMPLE SCRIPT TO DEMONSTRATE HOW TO      ####
+#### SUBMIT A STANDARD monc JOB WITHOUT A RESTART         ####
+#### Script assumes that there are directories called     ####
+#### monc_stdout , checkpoint_files and diagnostic_files  ####
+#### If these do not exist, MONC will fail                ####
+##############################################################
+
+echo Starting job
+
+module purge
+module load user
+module switch intel gnu/8.3.0
+#module switch openmpi mvapich2
+module load netcdf hdf5 fftw fcm
+
+module list
+
+if [ ! -d checkpoint_files ]; then mkdir checkpoint_files; fi
+if [ ! -d monc_stdout ]; then mkdir monc_stdout; fi
+if [ ! -d diagnostic_files ]; then mkdir diagnostic_files; fi
+
+# MVAPICH2 RUNTIME VARIABLES 
+#MONC_THREAD_MULTIPLE=0
+#MV2_ENABLE_AFFINITY=0
+#MV2_SHOW_CPU_BINDING=1
+#MV2_USE_THREAD_WARNING=0
+#export MONC_THREAD_MULTIPLE MV2_ENABLE_AFFINITY MV2_SHOW_CPU_BINDING \
+#       MV2_USE_THREAD_WARNING
+
+# Set the number of threads to 1
+#   This prevents any system libraries from automatically  
+#   using threading.
+export OMP_NUM_THREADS=1
+
+# set env variables for submission command
+config_path='testcases/tank_experiments/straka.mcf'
+checkpoint_fn="checkpoint_files/straka_dump.nc"
+
+mpirun -np 4 ./build/bin/monc_driver.exe --config=$config_path  --checkpoint_file=$checkpoint_fn &> monc_stdout/output_straka_dump_1