Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-46942: Raise UpstreamFailureNoWorkFound in ReprocessVisitImageTask when detector is missing a necessary calibration #77

Merged
merged 2 commits into from
Oct 18, 2024
Merged

DM-46942: Raise UpstreamFailureNoWorkFound in ReprocessVisitImageTask when detector is missing a necessary calibration #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ad59cb
Raise UpstreamFailureNoWorkFound on missing calib
laurenam Oct 17, 2024
73cffd5
Add test for missing calibs in reprocessVisitImage
laurenam Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions python/lsst/drp/tasks/reprocess_visit_image.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,24 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
assert not inputs, "runQuantum got more inputs than expected"

detector_summary = visit_summary.find(detector)
lines = []
if detector_summary is None:
msg = f"Detector {detector} not found in visit summary table; not reprocessing this exposure."
raise pipeBase.NoWorkFound(msg)
lines.append(" > no entry for the detector was found in the visit summary table")
else:
if detector_summary.psf is None:
lines.append(" > the PSF model for the detector is None")
if detector_summary.wcs is None:
lines.append(" > the WCS model for the detector is None")
if detector_summary.apCorrMap is None:
lines.append(" > the aperture correction model map for the detector is None")
if detector_summary.photoCalib is None:
lines.append(" > the photometric calibration model for the detector is None")

if lines:
msg = "\n".join(lines)
raise pipeBase.UpstreamFailureNoWorkFound(
f"Skipping reprocessing of detector {detector} because:\n{msg}"
)

# Specify the fields that `annotate` needs below, to ensure they
# exist, even as None.
Expand Down
83 changes: 72 additions & 11 deletions tests/test_reprocess_visit_image.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,21 @@
from lsst.drp.tasks.reprocess_visit_image import ReprocessVisitImageTask


def make_visit_summary(wcs=None, photo_calib=None, detector=42):
def make_visit_summary(summary=None, psf=None, wcs=None, photo_calib=None, detector=42):
"""Return a visit summary table with an entry for the given detector."""
schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
lsst.afw.image.ExposureSummaryStats.update_schema(schema)
summary = lsst.afw.table.ExposureCatalog(schema)
if summary is None:
schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
lsst.afw.image.ExposureSummaryStats.update_schema(schema)
summary = lsst.afw.table.ExposureCatalog(schema)
if summary.find(detector) is not None:
raise RuntimeError(f"Detector {detector} already exists in visit summary table, can't re-add it.")

record = summary.addNew()
record.setId(detector)
record.setApCorrMap(lsst.afw.image.ApCorrMap()),

if photo_calib is not None:
record.setPhotoCalib(photo_calib)
else:
record.setPhotoCalib(lsst.afw.image.PhotoCalib(10, 0.5))
record.setPsf(psf)
record.setPhotoCalib(photo_calib)

if wcs is not None:
record.setWcs(wcs)
Expand Down Expand Up @@ -130,7 +131,9 @@ def setUp(self):
task = lsst.meas.algorithms.SubtractBackgroundTask(config=config)
self.background = task.run(self.truth_exposure).background
self.visit_summary = make_visit_summary(
wcs=self.truth_exposure.wcs, photo_calib=self.truth_exposure.photoCalib
psf=self.truth_exposure.psf,
wcs=self.truth_exposure.wcs,
photo_calib=self.truth_exposure.photoCalib,
)

# A catalog that looks like the output of finalizeCharacterization,
Expand Down Expand Up @@ -239,6 +242,7 @@ def setUp(self):
# dataIds for fake data
butlerTests.addDataIdValue(self.repo, "detector", detector)
butlerTests.addDataIdValue(self.repo, "detector", detector + 1)
butlerTests.addDataIdValue(self.repo, "detector", detector + 2)
butlerTests.addDataIdValue(self.repo, "exposure", exposure0)
butlerTests.addDataIdValue(self.repo, "exposure", exposure1)
butlerTests.addDataIdValue(self.repo, "visit", visit)
Expand Down Expand Up @@ -286,6 +290,11 @@ def setUp(self):
self.visit1_id = self.repo.registry.expandDataId(
{"instrument": instrument, "visit": visit, "detector": detector + 1}
)
# Third id for testing on a detector that is in visitSummary but
# has missing calibs.
self.visit2_id = self.repo.registry.expandDataId(
{"instrument": instrument, "visit": visit, "detector": detector + 2}
)
self.visit_only_id = self.repo.registry.expandDataId({"instrument": instrument, "visit": visit})

# put empty data
Expand All @@ -296,12 +305,26 @@ def setUp(self):
background = lsst.afw.math.makeBackground(lsst.afw.image.ImageF(100, 100), control)
self.butler.put(lsst.afw.image.PhotoCalib(10), "initial_photoCalib_detector", self.visit_id)
self.butler.put(lsst.afw.image.PhotoCalib(10), "initial_photoCalib_detector", self.visit1_id)
self.butler.put(lsst.afw.image.PhotoCalib(10), "initial_photoCalib_detector", self.visit2_id)
self.butler.put(lsst.afw.math.BackgroundList(background), "initial_pvi_background", self.visit_id)
self.butler.put(lsst.afw.math.BackgroundList(background), "initial_pvi_background", self.visit1_id)
self.butler.put(lsst.afw.math.BackgroundList(background), "initial_pvi_background", self.visit2_id)
self.butler.put(lsst.afw.math.BackgroundList(background), "skyCorr", self.visit_id)
self.butler.put(lsst.afw.math.BackgroundList(background), "skyCorr", self.visit1_id)
self.butler.put(lsst.afw.math.BackgroundList(background), "skyCorr", self.visit2_id)
self.butler.put(lsst.afw.table.SourceCatalog().asAstropy(), "finalized_src_table", self.visit_only_id)
self.butler.put(make_visit_summary(detector=detector), "finalVisitSummary", self.visit_only_id)
# Make a simple single gaussian psf so that psf is not None in
# finalVisitSummary table which would result in
# UpstreamFailureNoWorkFound being raised in ReprocessVisitImageTask,
# which will be independently tested with self.visit2_id.
simple_psf = lsst.meas.algorithms.SingleGaussianPsf(11, 11, 2.0)
photo_calib = lsst.afw.image.PhotoCalib(10, 0.5)
visit_summary = make_visit_summary(psf=simple_psf, photo_calib=photo_calib, detector=detector)
# Add a detector with an entry, but some required calibs set to None.
visit_summary = make_visit_summary(
summary=visit_summary, psf=None, photo_calib=None, detector=detector + 2
)
self.butler.put(visit_summary, "finalVisitSummary", self.visit_only_id)

def tearDown(self):
self.repo_path.cleanup()
Expand Down Expand Up @@ -376,7 +399,45 @@ def test_runQuantum_no_detector_in_visit_summary(self):
"background": self.visit1_id,
},
)
with self.assertRaisesRegex(lsst.pipe.base.NoWorkFound, "Detector 43 not found"):
msg = " > no entry for the detector was found in the visit summary table"
with self.assertRaisesRegex(
lsst.pipe.base.UpstreamFailureNoWorkFound, f"Skipping reprocessing of detector 43 because:\n{msg}"
):
lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum)

def test_runQuantum_missing_calibs_for_detector_in_visit_summary(self):
"""Test how the task handles the detector not being in the input visit
summary.
"""
task = ReprocessVisitImageTask()
lsst.pipe.base.testUtils.assertValidInitOutput(task)

quantum = lsst.pipe.base.testUtils.makeQuantum(
task,
self.butler,
self.visit2_id,
{
"exposures": [self.exposure0_id],
"visit_summary": self.visit_only_id,
"initial_photo_calib": self.visit2_id,
"background_1": self.visit2_id,
"background_2": self.visit2_id,
"calib_sources": self.visit_only_id,
# outputs
"exposure": self.visit2_id,
"sources": self.visit2_id,
"sources_footprints": self.visit2_id,
"background": self.visit2_id,
},
)
lines = [
" > the PSF model for the detector is None",
" > the photometric calibration model for the detector is None",
]
msg = "\n".join(lines)
with self.assertRaisesRegex(
lsst.pipe.base.UpstreamFailureNoWorkFound, f"Skipping reprocessing of detector 44 because:\n{msg}"
):
lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum)

def test_runQuantum_no_sky_corr(self):
Expand Down
Loading