DM-42201: Add flat core analysis tools pipeline

pipelines/_ingredients/verifyFlat.yaml

@@ -1,4 +1,9 @@
 description: cp_verify FLAT calibration verification
+imports:
+  - location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml
+    include:
+      - analyzeFlatCore
+      - analyzeFlatDetMergeCore
 tasks:
   verifyFlatIsr:
     class: lsst.ip.isr.IsrTask
@@ -17,6 +22,16 @@ tasks:
       connections.outputStats: "verifyFlatDetStats"
       connections.outputResults: "verifyFlatDetResults"
       useIsrStatistics: true
+  verifyFlatDetMerge:
+    class: lsst.cp.verify.CpVerifyDetMergeByFilterTask
+    config:
+      connections.inputStats: "verifyFlatDetStats"
+      connections.inputResults: "verifyFlatDetResults"
+      connections.outputStats: "verifyFlatDetMergeStats"
+      connections.outputResults: "verifyFlatDetMergeResults"
+      hasInputResults: true
+      hasMatrixCatalog: false
+      stageName: "flat"
   verifyFlatExp:
     class: lsst.cp.verify.CpVerifyFlatExpMergeTask
     config:
@@ -36,9 +51,21 @@ tasks:
       connections.outputResults: "verifyFlatResults"
       hasInputResults: true
       hasMatrixCatalog: false
+  analyzeFlatCore:
+    class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
+    config:
+      connections.data: "verifyFlatResults"
+      connections.outputName: "verifyFlatAnalysis"
+  analyzeFlatDetMergeCore:
+    class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
+    config:
+      connections.data: "verifyFlatDetMergeResults"
+      connections.outputName: "verifyFlatDetMergeAnalysis"
 contracts:
   - verifyFlatDet.connections.inputExp == verifyFlatIsr.connections.outputExposure
   - verifyFlatExp.connections.inputStats == verifyFlatDet.connections.outputStats
   - verifyFlatExp.connections.inputResults == verifyFlatDet.connections.outputResults
   - verifyFlat.connections.inputStats == verifyFlatExp.connections.outputStats
   - verifyFlat.connections.inputResults == verifyFlatExp.connections.outputResults
+  - analyzeFlatCore.connections.data == verifyFlat.connections.outputResults
+  - analyzeFlatDetMergeCore.connections.data == verifyFlatDetMerge.connections.outputResults

pipelines/_ingredients/verifyFlatLSST.yaml

@@ -1,4 +1,9 @@
 description: cp_verify FLAT calibration verification (LSST ISR task).
+imports:
+  - location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml
+    include:
+      - analyzeFlatCore
+      - analyzeFlatDetMergeCore
 tasks:
   verifyFlatIsr:
     class: lsst.ip.isr.IsrTaskLSST
@@ -30,6 +35,16 @@ tasks:
       connections.outputStats: "verifyFlatDetStats"
       connections.outputResults: "verifyFlatDetResults"
       useIsrStatistics: true
+  verifyFlatDetMerge:
+    class: lsst.cp.verify.CpVerifyDetMergeByFilterTask
+    config:
+      connections.inputStats: "verifyFlatDetStats"
+      connections.inputResults: "verifyFlatDetResults"
+      connections.outputStats: "verifyFlatDetMergeStats"
+      connections.outputResults: "verifyFlatDetMergeResults"
+      hasInputResults: true
+      hasMatrixCatalog: false
+      stageName: "flat"
   verifyFlatExp:
     class: lsst.cp.verify.CpVerifyFlatExpMergeTask
     config:
@@ -49,9 +64,21 @@ tasks:
       connections.outputResults: "verifyFlatResults"
       hasInputResults: true
       hasMatrixCatalog: false
+  analyzeFlatCore:
+    class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
+    config:
+      connections.data: "verifyFlatResults"
+      connections.outputName: "verifyFlatAnalysis"
+  analyzeFlatDetMergeCore:
+    class: lsst.analysis.tools.tasks.VerifyCalibDetectorTaskByFilter
+    config:
+      connections.data: "verifyFlatDetMergeResults"
+      connections.outputName: "verifyFlatDetMergeAnalysis"
 contracts:
   - verifyFlatDet.connections.inputExp == verifyFlatIsr.connections.outputExposure
   - verifyFlatExp.connections.inputStats == verifyFlatDet.connections.outputStats
   - verifyFlatExp.connections.inputResults == verifyFlatDet.connections.outputResults
   - verifyFlat.connections.inputStats == verifyFlatExp.connections.outputStats
   - verifyFlat.connections.inputResults == verifyFlatExp.connections.outputResults
+  - analyzeFlatCore.connections.data == verifyFlat.connections.outputResults
+  - analyzeFlatDetMergeCore.connections.data == verifyFlatDetMerge.connections.outputResults

python/lsst/cp/verify/configs/report.yaml

@@ -51,6 +51,18 @@ stages:
       description: "Processing configuration for the combined flat construction."
     verifyFlatApply_config:
       description: "Processing configuration for combined flat verification."
+    cpFlatCore_flatMeanPerAmp_FocalPlaneGeometryPlot:
+      description: "Focal plane mosaic of the combined flat means per amp."
+    cpFlatCore_flatNoisePerAmp_FocalPlaneGeometryPlot:
+      description: "Focal plane mosaic of the combined flat noise per amp."
+    cpFlatDetMergeCore_flatTestsByDate_GridPlot:
+      description: "Verfiy flat noise boolean by MJD per amp."
+    cpFlatDetMergeCore_flatMeansByDate_GridPlot:
+      description: "Flat mean by MJD per amp."
+    cpFlatDetMergeCore_flatNoiseByDate_GridPlot:
+      description: "Flat noise by MJD per amp."
+    cpFlatDetMergeCore_flatNoiseByMean_GridPlot:
+      description: "Verify flat noise by mean per amp."
   defects:
     defects:
       description: "The combined defects calibration."

python/lsst/cp/verify/mergeResults.py

@@ -27,6 +27,8 @@
 
 __all__ = ['CpVerifyExpMergeConfig', 'CpVerifyExpMergeTask',
            'CpVerifyRunMergeConfig', 'CpVerifyRunMergeTask',
+           'CpVerifyDetMergeConfig', 'CpVerifyDetMergeTask',
+           'CpVerifyDetMergeByFilterConfig', 'CpVerifyDetMergeByFilterTask',
            'CpVerifyExpMergeByFilterConfig', 'CpVerifyExpMergeByFilterTask',
            'CpVerifyRunMergeByFilterConfig', 'CpVerifyRunMergeByFilterTask',
            'CpVerifyVisitExpMergeConfig', 'CpVerifyVisitExpMergeTask',
@@ -477,6 +479,172 @@ class CpVerifyRunMergeTask(CpVerifyExpMergeTask):
 # End ExpMerge/RunMerge
 
 
+class CpVerifyDetMergeConnections(pipeBase.PipelineTaskConnections,
+                                  dimensions={"instrument", "detector"},
+                                  defaultTemplates={}):
+    inputStats = cT.Input(
+        name="detectorStats",
+        doc="Input statistics to merge.",
+        storageClass="StructuredDataDict",
+        dimensions=["instrument", "exposure", "detector"],
+        multiple=True,
+    )
+    inputResults = cT.Input(
+        name="detectorResults",
+        doc="Input results to merge.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "exposure", "detector"],
+        multiple=True,
+    )
+    inputMatrix = cT.Input(
+        name="detectorMatrix",
+        doc="Input matrix to merge.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "exposure", "detector"],
+        multiple=True,
+    )
+    camera = cT.PrerequisiteInput(
+        name="camera",
+        storageClass="Camera",
+        doc="Input camera.",
+        dimensions=["instrument", ],
+        isCalibration=True,
+    )
+
+    outputStats = cT.Output(
+        name="exposureStats",
+        doc="Output statistics.",
+        storageClass="StructuredDataDict",
+        dimensions=["instrument", "detector"],
+    )
+    outputResults = cT.Output(
+        name="exposureResults",
+        doc="Output results.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "detector"],
+    )
+    outputMatrix = cT.Output(
+        name="exposureMatrix",
+        doc="Output matrix.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "detector"],
+    )
+
+    def __init__(self, *, config=None):
+        super().__init__(config=config)
+
+        if not self.config.hasMatrixCatalog:
+            self.inputs.remove("inputMatrix")
+            self.outputs.remove("outputMatrix")
+        if not self.config.hasInputResults:
+            self.inputs.remove("inputResults")
+        if self.config.doDropStats:
+            self.outputs.remove("outputStats")
+
+
+class CpVerifyDetMergeConfig(CpVerifyExpMergeConfig,
+                             pipelineConnections=CpVerifyDetMergeConnections):
+    """Configuration paramters for exposure stats merging.
+    """
+    mergeDimension = pexConfig.Field(
+        dtype=str,
+        doc="Dimension name for this input.",
+        default="detector",
+    )
+
+
+class CpVerifyDetMergeTask(CpVerifyExpMergeTask):
+    """Merge statistics from detectors together.
+    """
+    ConfigClass = CpVerifyDetMergeConfig
+    _DefaultName = 'cpVerifyDetMerge'
+
+    pass
+
+
+class CpVerifyDetMergeByFilterConnections(pipeBase.PipelineTaskConnections,
+                                          dimensions={"instrument", "detector", "physical_filter"},
+                                          defaultTemplates={}):
+    inputStats = cT.Input(
+        name="detectorStats",
+        doc="Input statistics to merge.",
+        storageClass="StructuredDataDict",
+        dimensions=["instrument", "exposure", "detector", "physical_filter"],
+        multiple=True,
+    )
+    inputResults = cT.Input(
+        name="detectorResults",
+        doc="Input results to merge.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "exposure", "detector", "physical_filter"],
+        multiple=True,
+    )
+    inputMatrix = cT.Input(
+        name="detectorMatrix",
+        doc="Input matrix to merge.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "exposure", "detector", "physical_filter"],
+        multiple=True,
+    )
+    camera = cT.PrerequisiteInput(
+        name="camera",
+        storageClass="Camera",
+        doc="Input camera.",
+        dimensions=["instrument", ],
+        isCalibration=True,
+    )
+
+    outputStats = cT.Output(
+        name="exposureStats",
+        doc="Output statistics.",
+        storageClass="StructuredDataDict",
+        dimensions=["instrument", "detector", "physical_filter"],
+    )
+    outputResults = cT.Output(
+        name="exposureResults",
+        doc="Output results.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "detector", "physical_filter"],
+    )
+    outputMatrix = cT.Output(
+        name="exposureMatrix",
+        doc="Output matrix.",
+        storageClass="ArrowAstropy",
+        dimensions=["instrument", "detector", "physical_filter"],
+    )
+
+    def __init__(self, *, config=None):
+        super().__init__(config=config)
+
+        if not self.config.hasMatrixCatalog:
+            self.inputs.remove("inputMatrix")
+            self.outputs.remove("outputMatrix")
+        if not self.config.hasInputResults:
+            self.inputs.remove("inputResults")
+        if self.config.doDropStats:
+            self.outputs.remove("outputStats")
+
+
+class CpVerifyDetMergeByFilterConfig(CpVerifyExpMergeConfig,
+                                     pipelineConnections=CpVerifyDetMergeByFilterConnections):
+    """Configuration paramters for exposure stats merging.
+    """
+    mergeDimension = pexConfig.Field(
+        dtype=str,
+        doc="Dimension name for this input.",
+        default="detector",
+    )
+
+
+class CpVerifyDetMergeByFilterTask(CpVerifyExpMergeTask):
+    """Merge statistics from detectors together.
+    """
+    ConfigClass = CpVerifyDetMergeByFilterConfig
+    _DefaultName = 'cpVerifyDetMergeByFilter'
+
+    pass
+
+
 # Begin ExpMergeByFilter/RunMergeByFilter
 class CpVerifyExpMergeByFilterConnections(pipeBase.PipelineTaskConnections,
                                           dimensions={"instrument", "exposure", "physical_filter"},