From ea70433da97a80087a476f139b1d8bd9d4fde18f Mon Sep 17 00:00:00 2001
From: Reijo Keskitalo <reijo.keskitalo@gmail.com>
Date: Mon, 7 Oct 2024 15:47:10 -0700
Subject: [PATCH] Use larger process groups to speed up sim

---
 .../phase1/sims/run.sat.mfhf.slurm            |  9 +++--
 .../phase1/sims/scale_maps.sat.py             | 39 ++++++++++---------
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/chile_optimization_sims/phase1/sims/run.sat.mfhf.slurm b/chile_optimization_sims/phase1/sims/run.sat.mfhf.slurm
index 1303a6a..36a5d76 100644
--- a/chile_optimization_sims/phase1/sims/run.sat.mfhf.slurm
+++ b/chile_optimization_sims/phase1/sims/run.sat.mfhf.slurm
@@ -1,7 +1,7 @@
 #!/bin/bash
 #SBATCH --qos=regular
 #SBATCH --time=01:00:00
-#SBATCH --nodes=4
+#SBATCH --nodes=8
 #SBATCH --job-name=CMBS4_phase1
 #SBATCH --licenses=SCRATCH
 #SBATCH --constraint=cpu
@@ -18,7 +18,7 @@ export HDF5_USE_FILE_LOCKING=FALSE
 export TOAST_FUNCTIME=1
 
 # Parallelization
-export OMP_NUM_THREADS=8
+export OMP_NUM_THREADS=2
 export OMP_PLACES=threads
 export OMP_PROC_BIND=spread
 let nnode=$SLURM_JOB_NUM_NODES
@@ -26,7 +26,7 @@ let nnode=$SLURM_JOB_NUM_NODES
 let ntask_node=256/$OMP_NUM_THREADS
 let ntask=$nnode*$ntask_node
 let ncore=$OMP_NUM_THREADS
-let groupsize=1
+let groupsize=8
 
 echo "Start: " `date`
 echo "Running with"
@@ -62,7 +62,7 @@ for band in f085 f095 f145 f155 f220 f280; do
             # args+=" --thinfp 16"
 	    ;;
 	f220|f280)
-	    fpfile=../focalplanes/focalplane_SAT1_SAT_${band}_ST1.h5
+	    fpfile=../focalplanes/focalplane_SAT1_SAT_${band}_ST2.h5
 	    pwv_limit=2.0
             # args+=" --thinfp 16"
 	    ;;
@@ -93,6 +93,7 @@ for band in f085 f095 f145 f155 f220 f280; do
     echo "${args[@]}"
     srun -N $nnode -n $ntask -c $ncore --cpu_bind=cores \
          toast_sim_ground.py ${args[@]} \
+         --binner.sync_type "allreduce" \
          >& $logfile
     
     exit
diff --git a/chile_optimization_sims/phase1/sims/scale_maps.sat.py b/chile_optimization_sims/phase1/sims/scale_maps.sat.py
index b82d5c0..4ffc61f 100644
--- a/chile_optimization_sims/phase1/sims/scale_maps.sat.py
+++ b/chile_optimization_sims/phase1/sims/scale_maps.sat.py
@@ -57,7 +57,7 @@
 # f_weather from the simulation logs on 10/07/2024
 
 cut_3mm = 1 - 903 / 5157
-cut_2mm = None
+cut_2mm = 1 - 1525 / 5157
 f_weather_sim["sat"] = {
     "f030" : cut_3mm,
     "f040" : cut_3mm,
@@ -84,19 +84,20 @@
 
 # Number of telescope years is a parameter
 
-ntube_lf = 3
-ntube_mf = 9
-ntube_hf = 6
+nsat = 9
+ntube_lf = nsat // 3
+ntube_mf = nsat
+ntube_hf = nsat // 3 * 2
 nyear = 10
 n_years = {
-    "f030" : [nyear * ntube_lf],
-    "f040" : [nyear * ntube_lf],
-    "f085" : [nyear * ntube_mf],
-    "f095" : [nyear * ntube_mf],
-    "f145" : [nyear * ntube_mf],
-    "f155" : [nyear * ntube_mf],
-    "f220" : [nyear * ntube_hf],
-    "f280" : [nyear * ntube_hf],
+    "f030" : nyear * ntube_lf,
+    "f040" : nyear * ntube_lf,
+    "f085" : nyear * ntube_mf,
+    "f095" : nyear * ntube_mf,
+    "f145" : nyear * ntube_mf,
+    "f155" : nyear * ntube_mf,
+    "f220" : nyear * ntube_hf,
+    "f280" : nyear * ntube_hf,
 }
 
 
@@ -105,7 +106,7 @@
 for flavor in "sat",:
     nrow, ncol = 2, 4
     fig = plt.figure(figsize=[4 * ncol, 4 * nrow])
-    fig.suptitle(f"{flavor} {n_year} years")
+    fig.suptitle(f"{flavor}, {nsat * 3} tubes")
     iplot = 0
     for band in f_total[flavor].keys():
         fname_in = f"outputs/{flavor}/{band}/mapmaker_cov.fits"
@@ -169,7 +170,7 @@
             depth,
             min=vmin,
             max=vmax,
-            title=band,
+            title=f"{band}, {n_year} tube years",
             sub=[nrow, ncol, iplot],
             cmap="inferno",
             unit="$\mu$K.arcmin",
@@ -177,9 +178,9 @@
             format="%.3f"
         )
 
-        # Save plot
+    # Save plot
 
-        os.makedirs("plots", exist_ok=True)
-        fname_plot = f"plots/{flavor}_{n_year}years.png"
-        fig.savefig(fname_plot)
-        print(f"Plot saved in {fname_plot}")
+    os.makedirs("plots", exist_ok=True)
+    fname_plot = f"plots/{flavor}.png"
+    fig.savefig(fname_plot)
+    print(f"Plot saved in {fname_plot}")