Add multilayer preproc (#1026)

* add preliminary multilayer preprocessing pipeline

* refactor multilayer_preprocess_tod, move common processing funcs into common file

* Add cfg checker to pp util

* Align multilayer with pp util instead of sp util

* Populates reference information in dependent database in multilayer.

* add multilayer_load_and_preprocess

* delete preprocess_common.py from site_pipeline

* update multilayer_load_and_preprocess

* add multilayer_preproc_or_load_group

* refactor preprocess_tod, multilayer_preprocess_tod, preproc_and_load_group

* many bug fixes, some refactoring

* fix typo

* address comments, add cleanup stage before running through obs

* Minor syntax edits.

* This adds the modifications required to run the mapmaker with multilayer preprocessing

* Adding a warning against single precision for maps

* Implementing save_group_and_cleanup into the mapmaker

* address missing entry problem, some other fixes

* fixing the save_group_and_cleanup block

* Fix to config check failures.

---------

Co-authored-by: Michael McCrackan <[email protected]>
Co-authored-by: Michael McCrackan <[email protected]>
Co-authored-by: Michael McCrackan <[email protected]>
Co-authored-by: Michael McCrackan <[email protected]>
Co-authored-by: Michael McCrackan <[email protected]>
Co-authored-by: chervias <[email protected]>

sotodlib/mapmaking/demod_mapmaker.py

@@ -447,8 +447,9 @@ def make_demod_map(context, obslist, noise_model, info,
         Noise model to pass to DemodMapmaker.
     info : list
         Information for the database, will be written as a .hdf file.
-    preprocess_config : dict
-        Dictionary with the config yaml file for the preprocess database.
+    preprocess_config : list of dict
+        List of dictionaries with the config yaml file for the preprocess database.
+        If two, then a multilayer preprocessing is to be used.
     prefix : str
         Prefix for the output files
     shape : tuple, optional
@@ -492,7 +493,7 @@ def make_demod_map(context, obslist, noise_model, info,
         List of outputs from preprocess database. To be used in cleanup_mandb.
     """
     from ..preprocess import preprocess_util
-    context = core.Context(context)
+    #context = core.Context(context)
     if L is None:
         L = preprocess_util.init_logger("Demod filterbin mapmaking")
     pre = "" if tag is None else tag + " "
@@ -508,14 +509,23 @@ def make_demod_map(context, obslist, noise_model, info,
     errors = [] ; outputs = []; # PENDING: do an allreduce of these.
                                 # not needed for atomic maps, but needed for
                                 # depth-1 maps
+    if len(preprocess_config)==1:
+        preproc_init = preprocess_config[0]
+        preproc_proc = None
+    else:
+        preproc_init = preprocess_config[0]
+        preproc_proc = preprocess_config[1]
+
     for oi in range(len(obslist)):
         obs_id, detset, band = obslist[oi][:3]
         name = "%s:%s:%s" % (obs_id, detset, band)
-        error, output, obs = preprocess_util.preproc_or_load_group(obs_id,
-                            configs=preprocess_config,
-                            dets={'wafer_slot':detset, 'wafer.bandpass':band}, 
-                            logger=L, context=context, overwrite=False)
-        errors.append(error) ; outputs.append(output) ;
+        error, output_init, output_proc, obs = preprocess_util.preproc_or_load_group(obs_id,
+                                                configs_init=preproc_init,
+                                                configs_proc=preproc_proc,
+                                                dets={'wafer_slot':detset, 'wafer.bandpass':band},
+                                                logger=L,
+                                                overwrite=False)
+        errors.append(error) ; outputs.append((output_init, output_proc)) ;
         if error not in [None,'load_success']:
             L.info('tod %s:%s:%s failed in the prepoc database'%(obs_id,detset,band))
             continue

sotodlib/preprocess/pcore.py

@@ -443,8 +443,12 @@ def run(self, aman, proc_aman=None, select=True, sim=False, update_plot=False):
         
         """
         if proc_aman is None:
-            proc_aman = core.AxisManager( aman.dets, aman.samps)
-            full = core.AxisManager( aman.dets, aman.samps)
+            if 'preprocess' in aman:
+                proc_aman = aman.preprocess
+                full = aman.preprocess
+            else:
+                proc_aman = core.AxisManager(aman.dets, aman.samps)
+                full = core.AxisManager( aman.dets, aman.samps)
             run_calc = True
             update_plot = False
         else: