feat: Add preprocessing PBS example and restructure Blue Waters runni…

…ng (#7)

* Add Drell-Yan preprocessing PBS script and Bash submission script
* Place default virtual environment first on PATH for Shifter jobs to ensure Python works as expected
   - `export PATH="/usr/local/venv/bin:${PATH}"`
* Restructure PBS job files by physics process for ease of file management and submission
* Restructure file output directory structure on Blue Waters to be /physics-process/action. e.g.:
   - drell-yan/madgraph/
   - drell-yan/delphes/

bluewaters/README.md

@@ -4,8 +4,8 @@
 
 To submit jobs to run on Portable Batch System (PBS) Torque, write a PBS submission file, and then submit the PBS job to Torque using `qsub`.
 
-For example, `drell-yan_madgraph5.pbs` defines all the commands required to run the example MadGraph5 simulation for Drell-Yan defined in `configs/madgraph5/drell-yan.mg5` on Shifter inside a specified container.
-So the PBS job file (`drell-yan_madgraph5.pbs`) is where you define what you want to have happen, and then you submit it with `qsub` just using a 1 line Bash script (`pbs_drell-yan_madgraph5.sh`) for easy of use and to make keeping track of workflows easier.
+For example, `drell-yan/madgraph5.pbs` defines all the commands required to run the example MadGraph5 simulation for Drell-Yan defined in `configs/madgraph5/drell-yan.mg5` on Shifter inside a specified container.
+So the PBS job file (`drell-yan/madgraph5.pbs`) is where you define what you want to have happen, and then you submit it with `qsub` just using a 1 command Bash script (`run_madgraph5.sh`) for easy of use and to make keeping track of workflows easier.
 
 Once jobs are submitted they can be tracked in the batch system with
 
@@ -19,15 +19,21 @@ qstat -u $USER
 * One would then submit the Drell-Yan job with
 
 ```console
-bash pbs_drell-yan_madgraph5.sh
+bash run_madgraph5.sh drell-yan
 ```
 
 to generate LHE and HEPMC2 simulation files
 
 * Then once the MadGraph5 and PYTHIA8 simulation is done, run DELPHES on the HEPMC2 input with
 
 ```console
-bash pbs_drell-yan_delphes.sh
+bash run_delphes.sh drell-yan
+```
+
+* To then run the preprocessing needed for MoMEMta to run, run
+
+```console
+bash run_preprocessing.sh drell-yan
 ```
 
 ## Interactive Session

bluewaters/drell-yan_delphes.pbs → bluewaters/drell-yan/delphes.pbs

@@ -11,8 +11,8 @@
 # Use shifter queue
 #PBS -l gres=shifter
 
-# Set the job name
-#PBS -N drell-yan_delphes
+# Set the PBS_JOBNAME
+#PBS -N delphes
 
 # Set the job stdout and stderr
 #PBS -e "${PBS_JOBNAME}.${PBS_JOBID}.err"
@@ -28,16 +28,12 @@
 # Ensure shifter enabled
 module load shifter
 
-OUTPUT_DIR="drell-yan_output"
-OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PBS_JOBNAME}/${PBS_JOBID}"
-OUTPUT_PATH="${OUTPUT_BASE_PATH}/${OUTPUT_DIR}"
-mkdir -p "${OUTPUT_BASE_PATH}"
-if [ -d "${OUTPUT_PATH}" ];then
-  echo " Run: rm -rf ${OUTPUT_PATH}"
-  exit 1
-fi
+PHYSICS_PROCESS="drell-yan"
+OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
+OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
+mkdir -p "${OUTPUT_PATH}"
 
-INPUT_FILE_PATH="/mnt/c/scratch/sciteam/${USER}/drell-yan/nevents_10e4_run/drell-yan_output/Events/run_01/tag_1_pythia8_events.hepmc"
+INPUT_FILE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/madgraph/nevents_10e4_run/Events/run_01/tag_1_pythia8_events.hepmc"
 
 # $HOME is /u/sciteam/${USER}
 SHIFTER_IMAGE="scailfin/delphes-python-centos:3.5.0"
@@ -59,6 +55,8 @@ aprun \
     --volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
     --workdir=/root/data \
     -- /bin/bash -c 'export LD_LIBRARY_PATH=$(echo -e "${LD_LIBRARY_PATH//\:/\\n}" | grep -v /opt/cray/nvidia/390.46-1_1.0502.2481.1.1.gem/lib64 | tr "\n" ":") && \
+        export PATH="/usr/local/venv/bin:${PATH}" && \
+        printf "\n# printenv:\n" && printenv && printf "\n\n" && \
         DelphesHepMC2 \
         /usr/local/venv/cards/delphes_card_ATLAS.tcl \
         delphes_output.root \

bluewaters/drell-yan_madgraph5.pbs → bluewaters/drell-yan/madgraph5.pbs

@@ -10,8 +10,8 @@
 # Use shifter queue
 #PBS -l gres=shifter
 
-# Set the job name
-#PBS -N drell-yan
+# Set the PBS_JOBNAME
+#PBS -N madgraph
 
 # Set the job stdout and stderr
 #PBS -e "${PBS_JOBNAME}.${PBS_JOBID}.err"
@@ -27,13 +27,10 @@
 # Ensure shifter enabled
 module load shifter
 
-OUTPUT_DIR="drell-yan_output"
-OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PBS_JOBNAME}/${PBS_JOBID}"
-OUTPUT_PATH="${OUTPUT_BASE_PATH}/${OUTPUT_DIR}"
-mkdir -p "${OUTPUT_BASE_PATH}"
-if [ -d "${OUTPUT_PATH}" ];then
-  rm -rf "${OUTPUT_PATH}"
-fi
+PHYSICS_PROCESS="drell-yan"
+OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
+OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
+mkdir -p "${OUTPUT_PATH}"
 
 # $HOME is /u/sciteam/${USER}
 SHIFTER_IMAGE="neubauergroup/bluewaters-mg5_amc:3.1.1"
@@ -48,4 +45,6 @@ aprun \
     --volume="${OUTPUT_BASE_PATH}":/root/data \
     --volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
     --workdir=/root/data \
-    -- /bin/bash -c 'mg5_aMC /mnt/a/'"${HOME}"'/MadGraph5-simulation-configs/configs/madgraph5/drell-yan.mg5'
+    -- /bin/bash -c 'export PATH="/usr/local/venv/bin:${PATH}" && \
+        printf "\n# printenv:\n" && printenv && printf "\n\n" && \
+        mg5_aMC /mnt/a/'"${HOME}"'/MadGraph5-simulation-configs/configs/madgraph5/drell-yan.mg5'

bluewaters/drell-yan/preprocessing.pbs

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Set the number of processing elements (PEs) or cores
+# Set the number of PEs per node
+#PBS -l nodes=1:ppn=8:xk
+
+# Set the wallclock time
+#PBS -l walltime=24:00:00
+
+# Use shifter queue
+#PBS -l gres=shifter
+
+# Set the PBS_JOBNAME
+#PBS -N preprocessing
+
+# Set the job stdout and stderr
+#PBS -e "${PBS_JOBNAME}.${PBS_JOBID}.err"
+#PBS -o "${PBS_JOBNAME}.${PBS_JOBID}.out"
+
+# Set email notification on termination or abort
+#PBS -m ea
+#PBS -M [email protected]
+
+# Set allocation to charge
+#PBS -A bbdz
+
+# Ensure shifter enabled
+module load shifter
+
+PHYSICS_PROCESS="drell-yan"
+OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
+OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
+mkdir -p "${OUTPUT_PATH}"
+
+# $HOME is /u/sciteam/${USER}
+SHIFTER_IMAGE="scailfin/delphes-python-centos:3.5.0"
+shifterimg pull "${SHIFTER_IMAGE}"
+
+INPUT_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/delphes/nevents_10e4/delphes_output.root"
+OUTPUT_FILE="${OUTPUT_PATH}/preprocessing_output_10e4.root"
+# The need to edit the contents of LD_LIBRARY_PATH is to remove NVIDIA libraries
+# that place versions of libOpenGL in LD_LIBRARY_PATH that conflict with the
+# Delphes Docker image and give a symbol lookup error.
+# c.f. https://bluewaters.ncsa.illinois.edu/shifter#remarks-on-running-apps
+# c.f. https://jira.ncsa.illinois.edu/browse/BWAPPS-7234
+aprun \
+  --bypass-app-transfer \
+  --pes-per-node 1 \
+  --cpu-binding none \
+  -- shifter \
+    --clearenv \
+    --image="${SHIFTER_IMAGE}" \
+    --volume="${OUTPUT_BASE_PATH}":/root/data \
+    --volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
+    --workdir=/root/data \
+    -- /bin/bash -c 'export LD_LIBRARY_PATH=$(echo -e "${LD_LIBRARY_PATH//\:/\\n}" | grep -v /opt/cray/nvidia/390.46-1_1.0502.2481.1.1.gem/lib64 | tr "\n" ":") && \
+        export PATH="/usr/local/venv/bin:${PATH}" && \
+        printf "\n# printenv:\n" && printenv && printf "\n\n" && \
+        cd /mnt/a/'"${HOME}"'/MadGraph5-simulation-configs/preprocessing && \
+        bash run_preprocessing.sh '"${INPUT_PATH}"' '"${OUTPUT_FILE}"

bluewaters/run_delphes.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+PROCESS_DIRECTORY="${1:-drell-yan}"
+qsub "${PROCESS_DIRECTORY}/delphes.pbs"

bluewaters/run_madgraph5.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+PROCESS_DIRECTORY="${1:-drell-yan}"
+qsub "${PROCESS_DIRECTORY}/madgraph5.pbs"

bluewaters/run_preprocessing.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+PROCESS_DIRECTORY="${1:-drell-yan}"
+qsub "${PROCESS_DIRECTORY}/preprocessing.pbs"