Skip to content

Commit

Permalink
feat: Improve PBS submissions scripts data flow (#9)
Browse files Browse the repository at this point in the history
* Revise drafts of PBS submission scripts for Blue Waters to make it easier to ingest the output of the previous stage and to provide better parameterization
* Add outpath and random seed CLI options to MadGraph5 config generator script
* Make MadGraph5 config generator script less location dependent by using absolute paths
* Update drell-yan.json config to use bbll topology
  • Loading branch information
matthewfeickert authored Aug 10, 2021
1 parent 9ec7c5c commit 0706408
Show file tree
Hide file tree
Showing 8 changed files with 116 additions and 36 deletions.
13 changes: 9 additions & 4 deletions bluewaters/drell-yan/delphes.pbs
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,16 @@
# Ensure shifter enabled
module load shifter

PHYSICS_PROCESS="drell-yan"
OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
PHYSICS_PROCESS="drell-yan_llbb"
INPUT_JOBID="12509809.bw"

USER_SCRATCH="/mnt/c/scratch/sciteam/${USER}"
OUTPUT_BASE_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
mkdir -p "${OUTPUT_PATH}"

INPUT_FILE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/madgraph/nevents_10e4_run/Events/run_01/tag_1_pythia8_events.hepmc"
# INPUT_FILE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/madgraph/nevents_10e4_run/Events/run_01/tag_1_pythia8_events.hepmc"
INPUT_FILE_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/madgraph/${INPUT_JOBID}/drell-yan_llbb_output/Events/run_01/tag_1_pythia8_events.hepmc"

# $HOME is /u/sciteam/${USER}
SHIFTER_IMAGE="scailfin/delphes-python-centos:3.5.0"
Expand All @@ -51,12 +55,13 @@ aprun \
-- shifter \
--clearenv \
--image="${SHIFTER_IMAGE}" \
--volume="${OUTPUT_BASE_PATH}":/root/data \
--volume="${OUTPUT_PATH}":/root/data \
--volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
--workdir=/root/data \
-- /bin/bash -c 'export LD_LIBRARY_PATH=$(echo -e "${LD_LIBRARY_PATH//\:/\\n}" | grep -v /opt/cray/nvidia/390.46-1_1.0502.2481.1.1.gem/lib64 | tr "\n" ":") && \
export PATH="/usr/local/venv/bin:${PATH}" && \
printf "\n# printenv:\n" && printenv && printf "\n\n" && \
if [ -f '"${INPUT_FILE_PATH}"'.gz ]; then gunzip '"${INPUT_FILE_PATH}"'.gz; fi && \
DelphesHepMC2 \
/usr/local/venv/cards/delphes_card_ATLAS.tcl \
delphes_output.root \
Expand Down
17 changes: 13 additions & 4 deletions bluewaters/drell-yan/madgraph5.pbs
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,15 @@
# Ensure shifter enabled
module load shifter

PHYSICS_PROCESS="drell-yan"
OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
PHYSICS_PROCESS="drell-yan_llbb"
USER_SCRATCH="/mnt/c/scratch/sciteam/${USER}"
OUTPUT_BASE_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
mkdir -p "${OUTPUT_PATH}"

CODE_BASE_PATH="/mnt/a/${HOME}/MadGraph5-simulation-configs"
RANDOM_SEED=300

# $HOME is /u/sciteam/${USER}
SHIFTER_IMAGE="neubauergroup/bluewaters-mg5_amc:3.1.1"
shifterimg pull "${SHIFTER_IMAGE}"
Expand All @@ -42,9 +46,14 @@ aprun \
-- shifter \
--clearenv \
--image="${SHIFTER_IMAGE}" \
--volume="${OUTPUT_BASE_PATH}":/root/data \
--volume="${OUTPUT_PATH}":/root/data \
--volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
--workdir=/root/data \
-- /bin/bash -c 'export PATH="/usr/local/venv/bin:${PATH}" && \
printf "\n# printenv:\n" && printenv && printf "\n\n" && \
mg5_aMC /mnt/a/'"${HOME}"'/MadGraph5-simulation-configs/configs/madgraph5/drell-yan.mg5'
python '"${CODE_BASE_PATH}"'/generate_config.py --help && \
echo "" && \
python '"${CODE_BASE_PATH}"'/generate_config.py -c '"${CODE_BASE_PATH}"'/configs/json/drell-yan.json --outpath $PWD --seed '"${RANDOM_SEED}"' && \
cat drell-yan_llbb.mg5 && \
echo "" && \
mg5_aMC drell-yan_llbb.mg5'
18 changes: 13 additions & 5 deletions bluewaters/drell-yan/momemta.pbs
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,23 @@
# Ensure shifter enabled
module load shifter

PHYSICS_PROCESS="drell-yan"
OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
TOPOLOGY="llbb"
PHYSICS_PROCESS="drell-yan_${TOPOLOGY}"
INPUT_JOBID="12509866.bw"
CODE_BASE_PATH="/mnt/a/${HOME}/MadGraph5-simulation-configs"

USER_SCRATCH="/mnt/c/scratch/sciteam/${USER}"
OUTPUT_BASE_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
mkdir -p "${OUTPUT_PATH}"

# $HOME is /u/sciteam/${USER}
SHIFTER_IMAGE="neubauergroup/bluewaters-momemta:1.0.1"
shifterimg pull "${SHIFTER_IMAGE}"

INPUT_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/preprocessing/12498179.bw/preprocessing_output_10e4.root"
# INPUT_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/preprocessing/${INPUT_JOBID}/preprocessing_output_10e4.root"
INPUT_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/preprocessing/${INPUT_JOBID}/preprocessing/preprocessing_output.root"

OUTPUT_FILE="${OUTPUT_PATH}/momemta_weights.root"

aprun \
Expand All @@ -46,11 +53,12 @@ aprun \
-- shifter \
--clearenv \
--image="${SHIFTER_IMAGE}" \
--volume="${OUTPUT_BASE_PATH}":/root/data \
--volume="${OUTPUT_PATH}":/root/data \
--volume="${USER_SCRATCH}/${PHYSICS_PROCESS}":"${USER_SCRATCH}/${PHYSICS_PROCESS}" \
--volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
--workdir=/root/data \
-- /bin/bash -c 'source scl_source enable devtoolset-8 && \
export PATH="/usr/local/venv/bin:${PATH}" && \
printf "\n# printenv:\n" && printenv && printf "\n\n" && \
cd /mnt/a/'"${HOME}"'/MadGraph5-simulation-configs/momemta/'"${PHYSICS_PROCESS}"' && \
cd '"${CODE_BASE_PATH}"'/momemta/'"${TOPOLOGY}"' && \
bash run_momemta.sh '"${INPUT_PATH}"' '"${OUTPUT_FILE}"
21 changes: 14 additions & 7 deletions bluewaters/drell-yan/preprocessing.pbs
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,23 @@
# Ensure shifter enabled
module load shifter

PHYSICS_PROCESS="drell-yan"
OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
PHYSICS_PROCESS="drell-yan_llbb"
INPUT_JOBID="12509845.bw"
CODE_BASE_PATH="/mnt/a/${HOME}/MadGraph5-simulation-configs"

USER_SCRATCH="/mnt/c/scratch/sciteam/${USER}"
OUTPUT_BASE_PATH="${USER_SCRATCH}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
mkdir -p "${OUTPUT_PATH}"

# $HOME is /u/sciteam/${USER}
SHIFTER_IMAGE="scailfin/delphes-python-centos:3.5.0"
shifterimg pull "${SHIFTER_IMAGE}"

INPUT_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/delphes/nevents_10e4/delphes_output.root"
OUTPUT_FILE="${OUTPUT_PATH}/preprocessing_output_10e4.root"
# INPUT_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/delphes/nevents_10e4/delphes_output.root"
INPUT_FILE="${USER_SCRATCH}/${PHYSICS_PROCESS}/delphes/${INPUT_JOBID}/delphes_output.root"
OUTPUT_FILE="preprocessing_output.root"

# The need to edit the contents of LD_LIBRARY_PATH is to remove NVIDIA libraries
# that place versions of libOpenGL in LD_LIBRARY_PATH that conflict with the
# Delphes Docker image and give a symbol lookup error.
Expand All @@ -50,11 +56,12 @@ aprun \
-- shifter \
--clearenv \
--image="${SHIFTER_IMAGE}" \
--volume="${OUTPUT_BASE_PATH}":/root/data \
--volume="${OUTPUT_PATH}":/root/data \
--volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
--workdir=/root/data \
-- /bin/bash -c 'export LD_LIBRARY_PATH=$(echo -e "${LD_LIBRARY_PATH//\:/\\n}" | grep -v /opt/cray/nvidia/390.46-1_1.0502.2481.1.1.gem/lib64 | tr "\n" ":") && \
export PATH="/usr/local/venv/bin:${PATH}" && \
printf "\n# printenv:\n" && printenv && printf "\n\n" && \
cd /mnt/a/'"${HOME}"'/MadGraph5-simulation-configs/preprocessing && \
bash run_preprocessing.sh '"${INPUT_PATH}"' '"${OUTPUT_FILE}"
cp -r '"${CODE_BASE_PATH}"'/preprocessing '"${OUTPUT_PATH}"' && \
cd preprocessing && \
bash run_preprocessing.sh '"${INPUT_FILE}"' '"${OUTPUT_FILE}"
1 change: 1 addition & 0 deletions bluewaters/run_madgraph5.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash

PROCESS_DIRECTORY="${1:-drell-yan}"
RANDOM_SEED=300
qsub "${PROCESS_DIRECTORY}/madgraph5.pbs"
14 changes: 10 additions & 4 deletions configs/json/drell-yan.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
{
"type": "madgraph5",
"process": "p p > l+ l-",
"output": "drell-yan",
"process": "p p > l+ l- b b~",
"output": "drell-yan_llbb",
"options": {
"shower": "Pythia8",
"nevents": 1000000
"madevent": {
"run_mode": 2
},
"run": {
"shower": "Pythia8",
"nevents": 10000,
"iseed": 0
}
},
"version": "0.1.0"
}
7 changes: 7 additions & 0 deletions configs/madgraph5/drell-yan.mg5
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
# MadEvent options
# 0=single core, 1=cluster, 2=multicore
set run_mode 2
set nb_core 6
generate p p > l+ l-
output drell-yan_output
launch drell-yan_output
# Set run card options
# Use PYTHIA8 for the shower/hadronization
shower=Pythia8
# Generate 1.0e+06 events
set nevents 1000000
# Set random seed
set iseed 0
61 changes: 49 additions & 12 deletions generate_config.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,41 @@
import json
import multiprocessing
from pathlib import Path

import click


def json_to_mg5(config):
mg5_str = f"generate {config['process']}"
mg5_str = ""

options = config.get("options", None)
if options:
madevent_options = options.get("madevent", None)
if madevent_options:
mg5_str += "# MadEvent options"
run_mode = madevent_options.get("run_mode", 2)
mg5_str += "\n# 0=single core, 1=cluster, 2=multicore"
mg5_str += f"\nset run_mode {run_mode}"
nb_core = madevent_options.get(
"nb_core", max(multiprocessing.cpu_count() - 2, 1)
)
mg5_str += f"\nset nb_core {nb_core}"

mg5_str += f"\ngenerate {config['process']}"
mg5_str += f"\noutput {config['output']}_output"
mg5_str += f"\nlaunch {config['output']}_output"

options = config["options"]
if "shower" in options:
run_options = options.get("run", None)
mg5_str += "\n# Set run card options"
if "shower" in run_options:
mg5_str += "\n # Use PYTHIA8 for the shower/hadronization"
mg5_str += f"\n shower={options['shower']}"
nevents = options.get("nevents", 1000)
mg5_str += f"\n shower={run_options['shower']}"
nevents = run_options.get("nevents", 1000)
mg5_str += f"\n # Generate {nevents:.1e} events"
mg5_str += f"\n set nevents {nevents}"
iseed = run_options.get("iseed", 0)
mg5_str += "\n # Set random seed"
mg5_str += f"\n set iseed {iseed}"
mg5_str += "\n"

return mg5_str
Expand All @@ -28,20 +48,37 @@ def json_to_mg5(config):
"config_path",
help="Path to JSON configuration.",
)
def generate_config(config_path):
config_path = Path().cwd().joinpath(config_path)
with open(config_path) as config_file:
@click.option(
"-o",
"--outpath",
"output_path",
help="Path to the output directory.",
)
@click.option(
"-s",
"--seed",
"random_seed",
type=int,
help="Random seed (iseed) for MadGraph5 run_card.dat.",
)
def generate_config(config_path, output_path, random_seed):
with open(Path(config_path).absolute()) as config_file:
config = json.load(config_file)

# Inject random seed if given at CLI
if random_seed:
config["options"]["run"]["iseed"] = random_seed

if config["type"] == "madgraph5":
output_config = json_to_mg5(config)
file_extension = ".mg5"

out_path = (
Path()
.cwd()
.joinpath("configs", config["type"], config["output"] + file_extension)
output_path = (
Path(output_path).absolute()
if output_path
else Path.cwd().joinpath("configs", config["type"])
)
out_path = output_path.joinpath(config["output"] + file_extension)
with open(out_path, "w") as outfile:
outfile.write(output_config)

Expand Down

0 comments on commit 0706408

Please sign in to comment.