Add DKRZ as a site to RTW (ESMValGroup#3837)

Co-authored-by: Alistair Sellar <[email protected]>

doc/sphinx/source/utils/RTW/user_guide/quick_start.rst

@@ -38,5 +38,17 @@ Quick Start Guide
         export CYLC_VERSION=8
         cylc vip -O jasmin
 
-* Browse the logs using `Cylc Review`_, a web service for browsing logs via an
-  HTTP interface.
+  * on DKRZ:
+
+    * add the following line to your ``~/.bashrc`` file to ensure the Cylc and
+      Rose executables can be found::
+
+        export PATH=/work/bd0854/metomi/bin:$PATH
+
+    * run the RTW on DKRZ::
+
+        cd ESMValTool/esmvaltool/utils/recipe_test_workflow
+        cylc vip -O dkrz
+
+* Optionally browse the logs using `Cylc Review`_,
+  a web service for browsing logs via an HTTP interface.

doc/sphinx/source/utils/RTW/user_guide/workflow.rst

@@ -25,9 +25,7 @@ The |RTW| performs the following steps:
      GitHub, or gets the latest container image from DockerHub and converts to
      a singularity image, depending on ``SITE``
   :Runs on:
-     Localhost (if cloning), or ``COMPUTE`` (if getting container), which
-     depends on the ``SITE``; on JASMIN, the ``get_esmval`` jobs will run on
-     LOTUS
+     Localhost, or ``COMPUTE`` on JASMIN
   :Executes:
      The ``clone_latest_esmval.sh`` script (if cloning), or a
      ``singularity build`` command (if getting container) from the |Rose| app
@@ -48,8 +46,7 @@ The |RTW| performs the following steps:
   :Description:
      Runs the requested recipes using |ESMValTool|
   :Runs on:
-     ``COMPUTE``, which depends on the ``SITE``; at the Met Office, the
-     ``process`` jobs will run on SPICE
+     ``COMPUTE``, which depends on the ``SITE``
   :Executes:
      The |ESMValTool| command line script from the |Rose| app
   :Details:
@@ -60,8 +57,7 @@ The |RTW| performs the following steps:
   :Description:
      Compares the output from the ``process`` job with |KGOs|
   :Runs on:
-     ``COMPUTE``, which depends on the ``SITE``; at the Met Office, the
-     ``compare`` jobs will run on SPICE
+     ``COMPUTE``, which depends on the ``SITE``
   :Executes:
      The :ref:`compare.py <compare_recipe_runs>` script from |ESMValTool|
      from the |Rose| app

esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf

@@ -0,0 +1,8 @@
+[command]
+default=singularity-env singularity build ${CONTAINER_PATH} ${DOCKER_SOURCE}
+
+[env]
+DOCKER_SOURCE=docker://esmvalgroup/esmvaltool:${ENV_NAME}
+
+[file:${CONTAINER_DIR}]
+mode=mkdir

esmvaltool/utils/recipe_test_workflow/app/install_env_file/opt/rose-app-dkrz.conf

@@ -0,0 +1,12 @@
+# Copy the site specific environment files to the 'share/bin' directory in the
+# installed Cylc workflow (this directory is automatically added to the
+# ${PATH} by Cylc).
+
+[command]
+default=BASH_XTRACEFD=1  # Send the output from 'set -x' to 'stdout' rather than 'stderr'.
+       =set -euxo pipefail
+       =cp ${ENV_FILE_SITE_PATH} ${SHARE_BIN_DIR}/${ENV_FILE}
+       =cp ${SINGULARITY_ENV_FILE_SITE_PATH} ${SHARE_BIN_DIR}/${SINGULARITY_ENV_FILE}
+
+[file:${SHARE_BIN_DIR}]
+mode=mkdir

esmvaltool/utils/recipe_test_workflow/flow.cylc

@@ -53,7 +53,6 @@
             ROSE_APP_OPT_CONF_KEYS = {{ SITE }}
 
     [[configure]]
-        pre-script = "mkdir -p ${USER_CONFIG_DIR}"
         [[[environment]]]
             ROSE_TASK_APP = configure
             DRS_CMIP3 = {{ DRS_CMIP3 }}

esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf

@@ -0,0 +1,10 @@
+[template variables]
+DRS_CMIP5="DKRZ"
+DRS_CMIP6="DKRZ"
+ENV_NAME="experimental"
+KGO_ROOT_PATH="/work/bd0854/KGO/esmvaltool_output/"
+ROOTPATH_CMIP5="/work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ"
+ROOTPATH_CMIP6="/work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ"
+ROOTPATH_OBS="/work/bd0854/DATA/ESMValTool2/OBS"
+ROOTPATH_OBS4MIPS="/work/bd0854/DATA/ESMValTool2/OBS"
+SITE="dkrz"

esmvaltool/utils/recipe_test_workflow/site/dkrz-env

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# USAGE dkrz-env COMMAND
+#
+# ENVIRONMENT
+#   ENV_NAME            The name of the container being used
+#   QUIET_MODE          Don't print confirmation messages
+#   PYTHONPATH_PREPEND  The path to prepend to PYTHONPATH
+#
+# OPTIONS
+#   COMMAND             The command to execute with options
+set -eu
+
+# Must be run before importing numpy, see
+# https://docs.dask.org/en/stable/array-best-practices.html#avoid-oversubscribing-threads
+export OMP_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export VECLIB_MAXIMUM_THREADS=1
+export NUMEXPR_NUM_THREADS=1
+
+# Ensure '~/.local' isn't added to 'sys.path'.
+export PYTHONNOUSERSITE=True
+
+WORKFLOW_RUN_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/bin"
+WORKFLOW_SHARE_BIN_DIR="${CYLC_WORKFLOW_SHARE_DIR}/cycle/bin"
+ROSE_APP_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/app/${ROSE_TASK_APP:-$CYLC_TASK_NAME}/bin"
+
+# Bind paths for container.
+export SINGULARITY_BIND="/home/b,/work,/scratch/b"
+
+# Suppress an ESMValTool "file not found" warning.
+export SINGULARITYENV_PROJ_DATA="/opt/conda/envs/esmvaltool/share/proj"
+
+# Provide mkfile needed to build esmfpy package.
+export SINGULARITYENV_ESMFMKFILE="/opt/conda/envs/esmvaltool/lib/esmf.mk"
+
+# Ensure that `singularity exec` finds the right version of python.
+export SINGULARITYENV_PREPEND_PATH="/opt/conda/envs/esmvaltool/bin"
+
+# Include Rose/Cylc workflow directories in container PATH.
+export SINGULARITYENV_APPEND_PATH="${WORKFLOW_RUN_BIN_DIR}:${WORKFLOW_SHARE_BIN_DIR}:${ROSE_APP_BIN_DIR}"
+
+# If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the
+# Python environment.
+if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then
+    echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}"
+    export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-}
+fi
+
+if [[ -z ${QUIET_MODE:-} ]]; then
+    echo "[INFO] Using the ${ENV_NAME} container"
+fi
+
+singularity_command="singularity-env singularity -q exec ${CONTAINER_PATH} $@"
+command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time ${singularity_command}"
+exec ${command}

esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# USAGE dkrz-singularity-env COMMAND
+#
+# OPTIONS
+#   COMMAND             The command to execute with options
+#
+# Since DKRZ uses the module environment to dynamically modify a user's
+# environment via modulefiles (more details are available at
+# https://docs.dkrz.de/doc/levante/access-and-environment.html#module-environment)
+# this additional environment file is required to enable access to singularity
+# on DKRZ. This file is used directly in the 'get_esmval' app for DKRZ, as well
+# as in the 'dkrz-env' environment file.
+set -eu
+
+module_count(){
+    module list -t 2>&1 | wc -l
+}
+
+safe_load(){
+    PRE_LOAD_COUNT=$(module_count)
+
+    module load "${1}"
+    # Check module count to determine whether module load was successful.
+
+    if (( PRE_LOAD_COUNT == $(module_count) )); then
+        echo "[ERROR] Failed to load: ${1}"
+        exit 1
+    fi
+}
+safe_load "singularity"
+
+command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time $@"
+exec ${command}

esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc

@@ -0,0 +1,102 @@
+#!jinja2
+[runtime]
+    [[root]]
+        [[[environment]]]
+            CONTAINER_DIR = ${ROSE_DATAC}/container
+            CONTAINER_FILE = esmvaltool.sif
+            CONTAINER_PATH = ${CONTAINER_DIR}/${CONTAINER_FILE}
+            # Warning: fragile assumption of location of esmvaltool code,
+            # issue #3437 contains more details.
+            ESMVALTOOL_DIR = /opt/conda/envs/esmvaltool/lib/python3.12/site-packages
+
+    # COMPUTE provides defaults for computation-heavy tasks.
+    # Specific tasks below override some defaults, e.g. time & memory.
+    [[COMPUTE]]
+        platform = levante
+        execution time limit = PT3M
+        [[[directives]]]
+            --wckey = RTW
+            --account = bk1088
+            --partition = interactive
+            --mem = 15G
+
+    [[install_env_file]]
+        [[[environment]]]
+            ROSE_APP_OPT_CONF_KEYS = {{ SITE }}
+            SINGULARITY_ENV_FILE=singularity-env
+            SINGULARITY_ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}-singularity-env
+
+    [[get_esmval]]
+        platform = localhost
+        execution time limit = PT10M  # Actual: 3m06s on 2024-12-18.
+        [[[environment]]]
+            # Move the location of the singularity cache from the user's
+            # home area to the directory where the container is saved.
+            # This variable must be specified only for the 'get_esmval'
+            # task, otherwise, for some reason, subsequent steps take much
+            # longer to run than the resources specified.
+            SINGULARITY_CACHEDIR = ${CONTAINER_DIR}
+
+    [[configure]]
+        platform = localhost
+        execution time limit = PT2M  # Actual: 0m14s on 2024-12-18.
+        [[[directives]]]
+            --mem = 2G
+
+    # Resources for recipes that need more than the default. Both time and
+    # memory should be specified, in case the default changes.
+    # Variable (fast, medium) must be consistent with flow.cylc.
+    # Comment indicates example recorded usage on DKRZ.
+    [[process_examples--recipe_python]]
+        # Actual: 0m07s 2.1 GB on 2024-12-18.
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_albedolandcover]]
+        # Actual: 0m10s 3.2 GB on 2024-12-18.
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_autoassess_landsurface_soilmoisture]]
+        # Actual: 0m13s 3.8 GB on 2024-12-18.
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_consecdrydays]]
+        # Actual: 0m08s 2.3 GB on 2024-12-18.
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_ensclus]]
+        # Actual: 0m35s 2.0 GB on 2024-12-18.
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_heatwaves_coldwaves]]
+        # Actual: 0m30s 1.6 GB on 2024-12-18.
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_ocean_amoc]]
+        # Actual: Currently broken: No input files found for Dataset
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_ocean_multimap]]
+        # Actual: Currently broken: No input files found for Dataset
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 15G
+
+    [[process_recipe_radiation_budget]]
+        # Actual: Currently broken: No input files found for Dataset
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem =15G

esmvaltool/utils/recipe_test_workflow/site/jasmin.cylc

@@ -2,7 +2,8 @@
 [runtime]
     [[root]]
         [[[environment]]]
-            # Warning: fragile assumption of location of esmvaltool code, see #3437
+            # Warning: fragile assumption of location of esmvaltool code,
+            # issue #3437 contains more details.
             ESMVALTOOL_DIR = /opt/conda/envs/esmvaltool/lib/python3.11/site-packages
 
     # COMPUTE provides defaults for computation-heavy tasks.