From 8d42007d5fffbb322187bab3e1b1683582369bae Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Fri, 8 Nov 2024 09:09:11 -0500 Subject: [PATCH 1/6] Fix test collection name in delete and exists commands. --- sbn_survey_image_service/data/test/generate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sbn_survey_image_service/data/test/generate.py b/sbn_survey_image_service/data/test/generate.py index 450767a..370e017 100644 --- a/sbn_survey_image_service/data/test/generate.py +++ b/sbn_survey_image_service/data/test/generate.py @@ -183,7 +183,8 @@ def create_tables() -> None: def delete_data(session) -> None: """Delete test data from database.""" - (session.query(Image).filter(Image.collection == "test-collection").delete()) + (session.query(Image).filter(Image.collection == + "urn:nasa:pds:survey:test-collection").delete()) def exists(session) -> bool: @@ -196,7 +197,7 @@ def exists(session) -> bool: try: results: Any = ( session.query(Image).filter( - Image.collection == "test-collection").all() + Image.collection == "urn:nasa:pds:survey:test-collection").all() ) except OperationalError: return False From 57bed5eb39477779fc6576b0b4da8c224a3790d4 Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Fri, 8 Nov 2024 10:28:41 -0500 Subject: [PATCH 2/6] Replace fitscut with astropy --- _install_fitscut | 53 --- _setup_development.sh => _setup_development | 8 - _setup_production.sh | 8 - _tests => _test | 2 +- docs/adding-data.rst | 14 +- docs/development.rst | 60 ++- docs/install.rst | 37 +- docs/service.rst | 35 +- pyproject.toml | 6 +- readme.md | 4 +- sbn_survey_image_service/config/exceptions.py | 6 - sbn_survey_image_service/services/image.py | 351 ++++++++++++------ .../test/test_services.py | 16 +- tests/__init__.py | 0 tests/test_sth.py | 18 - 15 files changed, 328 insertions(+), 290 deletions(-) delete mode 100755 _install_fitscut rename _setup_development.sh => _setup_development (92%) rename _tests => _test (69%) delete mode 100644 tests/__init__.py delete mode 100644 tests/test_sth.py diff --git a/_install_fitscut b/_install_fitscut deleted file mode 100755 index 55f0a22..0000000 --- a/_install_fitscut +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -eu - -# install fitscut and required libraries - -# some guidance from https://github.com/spacetelescope/caldp/blob/master/scripts/caldp-install-fitscut - -CFITSIO_VERSION=3.49 -WCSTOOLS_VERSION=3.9.7 - -command -v libtool || (echo "libtool is required to install fitscut dependencies" && exit 1) - -pushd . - -mkdir -p build -cd build - -# cfitsio -test ! -e cfitsio-${CFITSIO_VERSION}.tar.gz && wget https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio-${CFITSIO_VERSION}.tar.gz -tar xzf cfitsio-${CFITSIO_VERSION}.tar.gz -cd cfitsio-${CFITSIO_VERSION} -./configure --prefix=$VIRTUAL_ENV && make && make funpack && make install -cd .. - -# wcs tools -test ! -e wcstools-${WCSTOOLS_VERSION}.tar.gz && wget http://tdc-www.harvard.edu/software/wcstools/wcstools-${WCSTOOLS_VERSION}.tar.gz -tar xzf wcstools-${WCSTOOLS_VERSION}.tar.gz -cd wcstools-${WCSTOOLS_VERSION} -make -mkdir -p ${VIRTUAL_ENV}/include/libwcs -install -t ${VIRTUAL_ENV}/include/libwcs libwcs/*.h -install -t ${VIRTUAL_ENV}/lib libwcs/*.a -install -t ${VIRTUAL_ENV}/bin bin/* -cd .. - -# best way to ensure correct jpeg library version? -test ! -e jpegsrc.v6b.tar.gz && wget --content-disposition https://sourceforge.net/projects/libjpeg/files/libjpeg/6b/jpegsrc.v6b.tar.gz/download -tar xzf jpegsrc.v6b.tar.gz -cd jpeg-6b -mkdir -p $VIRTUAL_ENV/man/man1 -test ! -e libtool && ln -s /usr/bin/libtool -./configure --enable-shared --prefix=$VIRTUAL_ENV && make && make install -cd .. - -# fitscut -export CFLAGS=-I${VIRTUAL_ENV}/include -export LDFLAGS=-L${VIRTUAL_ENV}/lib -export LD_RUN_PATH=${VIRTUAL_ENV}/lib -test ! -e master.zip && wget https://github.com/spacetelescope/fitscut/archive/master.zip -unzip -o master.zip -cd fitscut-master -./configure --prefix=$VIRTUAL_ENV && make && make install - -popd diff --git a/_setup_development.sh b/_setup_development similarity index 92% rename from _setup_development.sh rename to _setup_development index cdb40cf..439e28c 100755 --- a/_setup_development.sh +++ b/_setup_development @@ -59,14 +59,6 @@ ${reset_color}""" pip install --upgrade -q -q -q pip setuptools wheel pip install -e .[recommended,dev,test,docs] -if [[ ! -e $VIRTUAL_ENV/bin/fitscut ]]; then - echo -e """${cyan} - Installing fitscut -${reset_color} -""" - ./_install_fitscut -fi - ### Link git pre-commit-hook script ln -fs $PWD/_precommit_hook $PWD/.git/hooks/pre-commit diff --git a/_setup_production.sh b/_setup_production.sh index 533dabd..34f90b3 100644 --- a/_setup_production.sh +++ b/_setup_production.sh @@ -62,14 +62,6 @@ latest=$(git describe --tags "$(git rev-list --tags --max-count=1)") git checkout $latest pip install .[recommended] -if [[ ! -e $VIRTUAL_ENV/bin/fitscut ]]; then - echo -e """${cyan} - Installing fitscut -${reset_color} -""" - ./_install_fitscut -fi - ### Link git pre-commit-hook script ln -fs $PWD/_precommit_hook $PWD/.git/hooks/pre-commit diff --git a/_tests b/_test similarity index 69% rename from _tests rename to _test index fb09833..bb61486 100755 --- a/_tests +++ b/_test @@ -1,7 +1,7 @@ #! /bin/bash # Run test scripts -pytest --verbose tests sbn_survey_image_service \ +pytest --verbose sbn_survey_image_service \ --cov=sbn_survey_image_service \ --cov-report=html \ --remote-data \ diff --git a/docs/adding-data.rst b/docs/adding-data.rst index 1acd322..287076c 100644 --- a/docs/adding-data.rst +++ b/docs/adding-data.rst @@ -74,12 +74,10 @@ in ``sbn_survey_image_service/data/add.py``, e.g., ``pds4_image()``. For example, the NEAT survey PDS4 data set v1.0 does not have pixel scale in the label, so we have hard coded it into the ``pds4_image`` function. - -Caveats -------- - It is assumed that survey images are FITS-compatible with a World Coordinate -System defined for a standard sky reference frame (ICRS). The cutout service -uses the FITS header, not the PDS labels, to define the sub-frame. This is a -limitation from using fitscut. **fitscut is planned to be replaced in a future -version** to better support metadata sourced from the PDS labels. +System defined for a standard sky reference frame (ICRS). + +.. attention:: + + The cutout service uses the FITS header, not the PDS labels, to define the sub-frame. Sourcing the + WCS from the labels will be addressed in a future version. diff --git a/docs/development.rst b/docs/development.rst index 75ffcc3..5b8778b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,18 +1,60 @@ Development =========== -The bash script ``_setup_development`` will create and setup a virtual -environment for development. Use that now, or follow :doc:`install`, replacing -the package installation step as directed below. +The bash script ``_setup_development`` will create and setup a virtual environment for development. Use that now, or follow :doc:`install`, replacing the package installation step as directed below. -For development, install the package with the "dev", "test", and "docs" options. -Also, it is beneficial to install the package in "editable" mode with the "-e" -option to pip: +For development, install the package with the "dev", "test", and "docs" options. Also, it is beneficial to install the package in "editable" mode with the "-e" option to pip: .. code:: bash pip install -e .[recommended,dev,test,docs] -If you have ``nodemon`` globally installed, then you can develop your api code -and have it automatically update on changes by running ``sbnsis start --dev``. -Otherwise, just run ``python -m sbn_survey_image_service.app``. + +Development mode +---------------- + +If you have ``nodemon`` globally installed, then you can develop your api code and have it automatically update on changes by running ``sbnsis start --dev``. Otherwise, just run ``python -m sbn_survey_image_service.app``. + + +Testing +------- + +Testing requires a test data set (images and PDS4 labels). The test data set may be generated with the provided script: + +.. code:: + + $ python3 -m sbn_survey_image_service.data.test.generate + usage: generate.py [-h] [--path PATH] [--add] [--exists] [--delete] + [--no-create-tables] + + Add/delete test data to/from SBN Survey Image Service database. + + options: + -h, --help show this help message and exit + --path PATH directory to which to save test data files (default: + /sbnsurveys/src/sbn-survey-image- + service/data/test) + --add add/create test data set (default: False) + --exists test for the existence of the test data set (default: + False) + --delete delete test data files and database rows (default: + False) + --no-create-tables do not attempt to create missing database tables + (default: False) + +Run the `--add` option to generate the files and populate the database: + +.. warning:: + + `--add` will use the database configuration in your `.env` file. It isn't harmful to add the test data set to a production database, but this may not be desired. Testing data may be removed from the database with the `--delete` option. + +.. code:: + + $ python3 -m sbn_survey_image_service.data.test.generate --add + INFO:__main__:Creating ~400 images and labels. + INFO:SBN Survey Image Service:2024-11-08 09:06:47,245: Logging to /sbnsurveys/src/sbn-survey-image-service/logging/sbnsis.log + INFO:SBN Survey Image Service:2024-11-08 09:06:47,245: Searching directory /sbnsurveys/src/sbn-survey-image-service/data/test + INFO:SBN Survey Image Service:2024-11-08 09:06:47,912: Searched 1 directories, found 404 labels, 404 processed. + INFO:__main__:Created and added 404 test images and their labels to the database. + +The script `_test` will run the tests with `pytest`. Coverage reports will be saved in HTML format to `htmlcov/`. Tests requiring network access are run by default (using the --remote-data option to pytest). These tests require a running instance of the service populated with NEAT data set. \ No newline at end of file diff --git a/docs/install.rst b/docs/install.rst index 13564c5..e26e840 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,20 +1,27 @@ Installation and Setup ====================== -Select and install your choice of database backend, either sqlite3 or -postgresql. If using postgresql, it is recommended to have two separate users -with access: one that can perform database maintenance (e.g., INSERT and UPDATE -permissions), and another that will be limited to read-only access. +Select and install your choice of database backend, either sqlite3 or postgresql. If using postgresql, it is recommended to have two separate users with access: one that can perform database maintenance (e.g., INSERT and UPDATE permissions), and another that will be limited to read-only access. Requirements ------------ -All python requirements are managed by the pyproject.toml file and automatically -installed with pip below. In addition, -[libtool](https://www.gnu.org/software/libtool/) and fitscut are needed. Follow -your typical system installation instructions for libtool. The installation of -fitscut is described below. +All Python requirements are managed by the pyproject.toml file and automatically +installed with pip below. Major Python dependencies are: + +* astropy +* connexion +* gunicorn +* pds4_tools +* Pillow +* requests +* SQLAlchemy + +Other dependencies: + +* logrotate +* nodemon - for running in development mode SBN SIS @@ -48,18 +55,6 @@ dependencies: pip install .[recommended] -fitscut -------- - -The SIS uses the fitscut utility for generating cutouts and web (e.g., JPEG) -images. Build and install it to the virtual environment. There is a bash -script that can do this automatically for you: - -.. code:: bash - - bash _install_fitscut - - SIS configuration ----------------- diff --git a/docs/service.rst b/docs/service.rst index 907c9d1..671fcec 100644 --- a/docs/service.rst +++ b/docs/service.rst @@ -33,55 +33,40 @@ Start the service in production mode: sbnsis start -The app will launch as a background process with the gunicorn WSGI server. The -number of workers is controlled with the env variable -``LIVE_GUNICORN_INSTANCES``. If you have trouble getting gunicorn to work, -running in non-daemon mode may help with debugging: +The app will launch as a background process with the gunicorn WSGI server. The number of workers is controlled with the env variable ``LIVE_GUNICORN_INSTANCES``. If you have trouble getting gunicorn to work, running in non-daemon mode may help with debugging: .. code:: bash sbnsis start --no-daemon -See :doc:`adding-data` for instructions on how to add data to the database. The -service does not need to be running to add data. +See :doc:`adding-data` for instructions on how to add data to the database. The service does not need to be running to add data. -It is recommended that you make the gunicorn-powered server accessible to the -outside world by proxy-passing requests through an HTTPS-enabled web server like -Apache. +It is recommended that you make the gunicorn-powered server accessible to the outside world by proxy-passing requests through an HTTPS-enabled web server like Apache. REST API -------- -Whether running in development or deployment modes, the Swagger documentation -for the REST API is available at ``http://localhost:API_PORT/BASE_HREF/ui``, -where ``API_PORT`` and ``BASE_HREF`` is defined in your ``.env``. +Whether running in development or deployment modes, the Swagger documentation for the REST API is available at ``http://localhost:API_PORT/BASE_HREF/ui``, where ``API_PORT`` and ``BASE_HREF`` is defined in your ``.env``. Logging ------- -Application error and informational logging is sent to the standard error stream -(stderr) and the file specified by the ``SBNSIS_LOG_FILE`` environment variable. +Application error and informational logging is sent to the standard error stream (stderr) and the file specified by the ``SBNSIS_LOG_FILE`` environment variable. If this is set to use the `logging/` directory in the repository, then the `sbnsis` tool will be able to automatically rotate logs using `logrotate` and the `logging/logrotate.config` file. If `SBNSIS_LOG_FILE` is set to another location, the `logrotate.config` file should be edited, or else log rotation will not work. Successful requests will produce two log items: the parameters and the results as JSON-formatted strings. The items are linked by a randomly generated job ID: -Successful requests will produce two log items: the parameters and the results -as JSON-formatted strings. The items are linked by a randomly generated job ID: +.. code-bock:: text + INFO 2021-02-17 14:10:16,960: {"job_id": "013f7515aa074ee58ad5929c8391a366", "id": "urn:nasa:pds:gbo.ast.neat.survey:data_tricam:p20021023_obsdata_20021023113833a", "ra": 47.4495603, "dec": 32.9424075, "size": "5arcmin", "format": "fits", "download": true} + INFO 2021-02-17 14:10:18,339: {"job_id": "013f7515aa074ee58ad5929c8391a366", "filename": "/hylonome3/transient/tmpw8s8qj1b.fits", "download_filename": "20021023113833a.fit_47.4495632.94241_5arcmin.fits", "mime_type": "image/fits"} -``` -INFO 2021-02-17 14:10:16,960: {"job_id": "013f7515aa074ee58ad5929c8391a366", "id": "urn:nasa:pds:gbo.ast.neat.survey:data_tricam:p20021023_obsdata_20021023113833a", "ra": 47.4495603, "dec": 32.9424075, "size": "5arcmin", "format": "fits", "download": true} -INFO 2021-02-17 14:10:18,339: {"job_id": "013f7515aa074ee58ad5929c8391a366", "filename": "/hylonome3/transient/tmpw8s8qj1b.fits", "download_filename": "20021023113833a.fit_47.4495632.94241_5arcmin.fits", "mime_type": "image/fits"} -``` -OpenAPI errors (e.g., invalid parameter values from the user) are not logged. -Internal code errors will be logged with a code traceback. +OpenAPI errors (e.g., invalid parameter values from the user) are not logged. Internal code errors will be logged with a code traceback. Updating SBNSIS --------------- -For minor updates that only require a restart of the server, first tag a release -on Github with the new version, e.g., v0.3.3. Then, pull the updated code and -the new tag, upgrade the SIS, and restart the service: +For minor updates that only require a restart of the server, first tag a release on Github with the new version, e.g., v0.3.3. Then, pull the updated code and the new tag, upgrade the SIS, and restart the service: .. code:: bash diff --git a/pyproject.toml b/pyproject.toml index ffcadd6..bddc9ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,14 @@ readme = "readme.md" authors = [{ name = "Michael S. P. Kelley", email = "msk@astro.umd.edu" }] license = { text = "BSD 3-Clause License" } dynamic = ["version"] -requires-python = ">= 3.10" +requires-python = ">=3.11" dependencies = [ "astropy>=6.0", + "fsspec>=2024.10.0", + "aiohttp>=3.10", + "requests>=2.32", + "Pillow>=11.0", "connexion[flask,swagger-ui,uvicorn]~=3.0", "gunicorn~=21.2", "pds4_tools==1.3", diff --git a/readme.md b/readme.md index b79ae56..a3725cf 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,6 @@ The [SBN Survey Image Service](https://sbnsurveys.astro.umd.edu/api/ui) (SIS) is ## Code Features -- Uses [fitscut](https://github.com/spacetelescope/fitscut) for image cutouts and JPEG/PNG generation - Flask API layer - OpenAPI spec with Connexion and Swagger - Gunicorn for production deployment @@ -18,5 +17,4 @@ The [SBN Survey Image Service](https://sbnsurveys.astro.umd.edu/api/ui) (SIS) is ## Documentation -Documentation is provided in the ``/docs`` directory, and at -[readthedocs](https://sbn-survey-image-service.readthedocs.io). +Documentation is provided in the ``/docs`` directory, and at [readthedocs](https://sbn-survey-image-service.readthedocs.io). diff --git a/sbn_survey_image_service/config/exceptions.py b/sbn_survey_image_service/config/exceptions.py index 5e0b7be..8b26f26 100644 --- a/sbn_survey_image_service/config/exceptions.py +++ b/sbn_survey_image_service/config/exceptions.py @@ -39,12 +39,6 @@ class ParameterValueError(SBNSISException): code = 400 -class FitscutError(SBNSISException): - """Error processing data with fitscut.""" - - code = 500 - - class DatabaseError(SBNSISException): """Database error.""" diff --git a/sbn_survey_image_service/services/image.py b/sbn_survey_image_service/services/image.py index 92b997b..6825538 100644 --- a/sbn_survey_image_service/services/image.py +++ b/sbn_survey_image_service/services/image.py @@ -4,34 +4,233 @@ __all__ = ["image_query"] import os -import subprocess -from typing import List, Optional, Tuple +from copy import copy +import warnings +from enum import Enum +from typing import List, Tuple +from PIL import Image as PIL_Image from sqlalchemy.orm.exc import NoResultFound +import numpy as np from astropy.coordinates import Angle +import astropy.units as u +from astropy.io import fits +from astropy.nddata import Cutout2D +from astropy.coordinates import SkyCoord, Angle +from astropy.wcs import WCS, FITSFixedWarning +from astropy.visualization import ZScaleInterval from .database_provider import data_provider_session, Session from ..data import url_to_local_file, generate_cache_filename from ..models.image import Image -from ..config.exceptions import InvalidImageID, ParameterValueError, FitscutError +from ..config.exceptions import InvalidImageID, ParameterValueError from ..config.env import ENV -FORMATS = {"png": "--png", "jpeg": "--jpg"} + +class ImageFormat(Enum): + # parameter, Pillow Image format, file extension + FITS = ("fits", "fits", "fits") + JPEG = ("jpeg", "jpeg", "jpeg") + JPG = ("jpg", "jpeg", "jpeg") + PNG = ("png", "png", "png") + DEFAULT = (None, "fits", "fits") + + def __new__(cls, parameter, format, extension): + self = object.__new__(cls) + self._value_ = parameter + self.format = format + self.extension = extension + return self + + +class CutoutSpec: + """Cutout center and size. + + + Parameters + ---------- + ra, dec : float or None + Extract sub-frame around this position: J2000 right ascension and + declination in degrees. + + size : str or None + Sub-frame size in angular units, e.g., "5 arcmin". Parsed with + `astropy.units.Quantity`. Minimum 1 arcsec. + + """ + + MINUMUM_SIZE: Angle = Angle(1 * u.arcsec) + + def __init__(self, ra: float | None, dec: float | None, size: str | Angle | None): + self.ra: float | None = ra + self.dec: float | None = dec + self.normalize() + + self.size: Angle = ( + self.MINUMUM_SIZE if size is None else max( + self.MINUMUM_SIZE, Angle(size)) + ) + + def __str__(self) -> str: + if self.full_size: + return "full_size" + return "".join([str(x) for x in (self.ra, self.dec, self.size)]) + + @property + def full_size(self) -> bool: + """Returns ``True`` if ``ra`` or ``dec`` is ``None``.""" + return self.coords is None + + @property + def coords(self) -> SkyCoord | None: + """Center as a `SkyCoord` object, or ``None``, if not defined.""" + if any((self.ra is None, self.dec is None)): + return None + return SkyCoord(self.ra, self.dec, unit=(u.deg, u.deg)) + + def normalize(self) -> None: + """Fix RA between 0 and 360, Dec between -90 and 90.""" + + if self.ra is not None: + # RA 0 to 360 + self.ra = self.ra % 360 + + if self.dec is not None: + # Dec -90 to 90 + self.dec = min(max(self.dec, -90), 90) + + def cutout(self, url: str) -> str: + """Generate a cutout from URL. + + + Parameters + ---------- + url : str + The URL to the full-size image. + + + Returns + ------- + fn : str + The file name of the a FITS file cutout, or, if ``self.full_size`` + is True, the full-sized FITS image. + + """ + + if self.full_size: + return url_to_local_file(url) + + fits_image_path: str = generate_cache_filename(url, str(self), "fits") + + # file exists? done! + if os.path.exists(fits_image_path): + return fits_image_path + + # output data object + result: fits.HDUList = fits.HDUList() + + # use fsspec so that we only read (and decompress) the portions of the + # file that are needed for the cutout + data: fits.HDUList + options: dict = { + "cache": False, + "use_fsspec": True, + "lazy_load_hdus": True, + "fsspec_kwargs": {"block_size": 1024 * 512, "cache_type": "bytes"}, + } + with fits.open(url, **options) as data: + i: int = 0 + + header: fits.Header = copy(data[i].header) + + wcs: WCS + with warnings.catch_warnings(): + warnings.simplefilter( + "ignore", (fits.verify.VerifyWarning, FITSFixedWarning) + ) + wcs = WCS(header) + + cutout: Cutout2D = Cutout2D( + data[i].section, self.coords, self.size, wcs=wcs + ) + + header.update(cutout.wcs.to_header()) + + result.append(fits.PrimaryHDU(cutout.data, header)) + result.writeto(fits_image_path) + + os.chmod(fits_image_path, 33204) + + return fits_image_path + + +def filename_suffix(cutout_spec: CutoutSpec, format: ImageFormat) -> str: + """Generate the file name suffix based on query parameters. + + + If any of ra/dec/size are None, then only `format` is used. + + Parameters + ---------- + cutout_spec : CutoutSpec + The center and size of the cutout. + + format : ImageFormat + Returned image format: fits, png, jpeg + + + Returns + ------- + suffix : str + + """ + + suffix: str = "" + if not cutout_spec.full_size: + # attachment file name is based on coordinates and size + suffix = f'_{cutout_spec.ra:.5f}{cutout_spec.dec:+.5f}_{cutout_spec.size}' + + return f"{suffix}.{format.extension}" + + +def create_browse_image( + fits_image_path: str, + output_image_path: str, + format: ImageFormat, +) -> None: + """Create the browse (JPEG, PNG) image. + + + Parameters + ---------- + fits_image_path : str + The source FITS image file name. + + image_path : str + The file name of the output. + + format : ImageFormat + The format of the output (must be JPEG, JPG, or PNG). + + """ + + interval: ZScaleInterval = ZScaleInterval() + data: np.ndarray = fits.getdata(fits_image_path) + data = interval(data, clip=True) * 255 + image: PIL_Image = PIL_Image.fromarray(data.astype(np.uint8)) + image.save(output_image_path, format=format.format) def image_query( obs_id: str, - ra: Optional[float] = None, - dec: Optional[float] = None, - size: Optional[str] = None, - format: str = "fits", + ra: float | None = None, + dec: float | None = None, + size: str | None = None, + format: str | ImageFormat | None = None, ) -> Tuple[str, str]: """Query database for image file or cutout thereof. - For cutouts, fitscut may not be able to work with fpacked data. Edit code - branching below for files that must be funpacked first. - Temporary files are saved to the path specified by the environment variable SBNSIS_CUTOUT_CACHE and reused, if possible. @@ -49,7 +248,7 @@ def image_query( Sub-frame size in angular units, e.g., '5arcmin'. Parsed with `astropy.units.Quantity`. - format : str, optional + format : str or ImageFormat, optional Returned image format: fits, png, jpeg @@ -63,112 +262,47 @@ def image_query( """ - if format not in ["fits", "png", "jpeg"]: + cutout_spec: CutoutSpec = CutoutSpec(ra, dec, size) + + try: + format = ImageFormat(format) + except ValueError: raise ParameterValueError( "image_query format must be fits, png, or jpeg.") + im: Image session: Session - exc: Exception with data_provider_session() as session: + exc: Exception try: - im: Image = session.query(Image).filter( - Image.obs_id == obs_id).one() + im = session.query(Image).filter(Image.obs_id == obs_id).one() except NoResultFound as exc: raise InvalidImageID("Image ID not found in database.") from exc session.expunge(im) - # normalize coordinates - if ra is not None: - # RA 0 to 360 - ra = ra % 360 - - if dec is not None: - # Dec -90 to 90 - dec = min(max(dec, -90), 90) - # create attachment file name - suffix: str = "" - if not any((ra is None, dec is None, size is None)): - # attachment file name is based on coordinates and size - suffix = f'_{ra:.5f}{dec:+.5f}_{size.replace(" ", "")}' - download_filename: str = os.path.splitext( os.path.basename(im.image_url))[0] - download_filename += f"{suffix}.{format}" - - # was this file already generated? serve it! - image_path = generate_cache_filename( - im.image_url, obs_id, str(ra), str(dec), str(size), format - ) - if os.path.exists(image_path): - return image_path, download_filename - - # otherwise, get the data and process - source_image_path: str = url_to_local_file(im.image_url) - - cmd: List[str] = ["fitscut", "-f"] - - if FORMATS.get(format) is not None: - cmd.extend( - [ - str(FORMATS.get(format)), - # '--asinh-scale' - "--autoscale=1", - ] - ) - - if (ra is None) or (dec is None) or (size is None): - if format == "fits": - # full-frame fits image, we're done - return source_image_path, os.path.basename(im.image_url) - - # full-frame jpeg or png - cmd.append("--all") - else: - # cutout requested + download_filename += filename_suffix(cutout_spec, format) - # cutout size is between 1 and ENV.MAXIMUM_CUTOUT_SIZE - try: - size_deg: float = Angle(size).deg - except ValueError as exc: - raise ParameterValueError(str(exc)) + # generate the cutout, as needed + fits_image_path: str = cutout_spec.cutout(im.image_url) - size_pix: int = int( - min(max(size_deg / im.pixel_scale, 1), ENV.MAXIMUM_CUTOUT_SIZE) - ) + # FITS format? done! + if format == ImageFormat.FITS: + return fits_image_path, download_filename - # funpack before fitscut? - decompress: bool = False - extension: str = "" - if obs_id.startswith("urn:nasa:pds:gbo.ast.atlas.survey"): - decompress = True - extension = "image" - # curiously, fitscut does not have an issue with fpacked NEAT data - - if decompress: - source_image_path = _funpack(source_image_path, extension) - - cmd.extend( - [ - "--wcs", - f"-x {ra}", - f"-y {dec}", - f"-c {size_pix}", - f"-r {size_pix}", - ] - ) + # formulate the final image file name + image_path = generate_cache_filename( + im.image_url, str(cutout_spec), format.extension) - cmd.extend([source_image_path, image_path]) + # was this file already generated? serve it! + if os.path.exists(image_path): + return image_path, download_filename - try: - subprocess.check_output(cmd) - except subprocess.CalledProcessError as exc: - raise FitscutError( - f"""Error processing data. -Command line = {" ".join(cmd)} -Process returned: {exc.output}""" - ) from exc + # create the jpeg or png + create_browse_image(fits_image_path, image_path, format) # rw-rw-r-- # In [16]: (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) @@ -176,22 +310,3 @@ def image_query( os.chmod(image_path, 33204) return image_path, download_filename - - -def _funpack(filename, extension): - """Decompress an fpacked file and return the new file name.""" - - decompressed_filename = generate_cache_filename(filename, extension) - - if not os.path.exists(decompressed_filename): - cmd: List[str] = [ - "funpack", - "-E", - extension, - "-O", - decompressed_filename, - filename, - ] - subprocess.check_call(cmd) - - return decompressed_filename diff --git a/sbn_survey_image_service/test/test_services.py b/sbn_survey_image_service/test/test_services.py index 0d07cfc..eb2cfde 100644 --- a/sbn_survey_image_service/test/test_services.py +++ b/sbn_survey_image_service/test/test_services.py @@ -6,6 +6,7 @@ from sqlalchemy.orm.session import Session import numpy as np from astropy.io import fits +from astropy.coordinates import Angle from ..data.test import generate from ..data import generate_cache_filename @@ -61,10 +62,7 @@ def test_image_query_full_frame_jpg(): expected_path: str = generate_cache_filename( "file://" + os.path.join(ENV.TEST_DATA_PATH, "test-000023.fits"), - "urn:nasa:pds:survey:test-collection:test-000023", - "None", - "None", - "None", + "full_size", "jpeg", ) @@ -84,10 +82,7 @@ def test_image_query_full_frame_png(): expected_path: str = generate_cache_filename( "file://" + os.path.join(ENV.TEST_DATA_PATH, "test-000023.fits"), - "urn:nasa:pds:survey:test-collection:test-000023", - "None", - "None", - "None", + "full_size", "png", ) @@ -101,7 +96,7 @@ def test_image_query_full_frame_png(): def test_image_query_cutout(): ra: float = 0 dec: float = -25 - size: str = "1deg" + size: str = Angle("1deg") image_path: str download_filename: str @@ -115,7 +110,6 @@ def test_image_query_cutout(): expected_path: str = generate_cache_filename( "file://" + os.path.join(ENV.TEST_DATA_PATH, "test-000102.fits"), - "urn:nasa:pds:survey:test-collection:test-000102", str(ra), str(dec), str(size), @@ -128,7 +122,7 @@ def test_image_query_cutout(): assert image_path == expected_path assert ( download_filename - == f'test-000102_{+ra:.5f}{+dec:.5f}_{size.replace(" ", "")}.fits' + == f'test-000102_{+ra:.5f}{+dec:.5f}_{size}.fits' ) # inspect file, value should be -25 at the center diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_sth.py b/tests/test_sth.py deleted file mode 100644 index 5b4fb60..0000000 --- a/tests/test_sth.py +++ /dev/null @@ -1,18 +0,0 @@ -''' -Doc string -''' - - -class MyClass: - x = 5 - - -p1: MyClass = MyClass() -print(p1.x) - -# -x: int = 1 - - -def test_1() -> None: - assert 1 == 1 From fc49754720a04c96ccfbf89e5d71b5696d2467a6 Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Fri, 8 Nov 2024 11:21:02 -0500 Subject: [PATCH 3/6] Docfix: testing does not require a running production instance --- docs/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 5b8778b..be4857c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -57,4 +57,4 @@ Run the `--add` option to generate the files and populate the database: INFO:SBN Survey Image Service:2024-11-08 09:06:47,912: Searched 1 directories, found 404 labels, 404 processed. INFO:__main__:Created and added 404 test images and their labels to the database. -The script `_test` will run the tests with `pytest`. Coverage reports will be saved in HTML format to `htmlcov/`. Tests requiring network access are run by default (using the --remote-data option to pytest). These tests require a running instance of the service populated with NEAT data set. \ No newline at end of file +The script `_test` will run the tests with `pytest`. Coverage reports will be saved in HTML format to `htmlcov/`. Tests requiring network access are run by default (using the --remote-data option to pytest). \ No newline at end of file From 6d721d701b005e8ad66daadfd14d9d64785afa72 Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Fri, 8 Nov 2024 11:23:03 -0500 Subject: [PATCH 4/6] Remove empty submodule imports --- sbn_survey_image_service/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sbn_survey_image_service/__init__.py b/sbn_survey_image_service/__init__.py index 22a019e..40d4ea3 100644 --- a/sbn_survey_image_service/__init__.py +++ b/sbn_survey_image_service/__init__.py @@ -7,8 +7,6 @@ # make cache directory set umask from .config import exceptions from .config import env -from . import services -from . import models try: __version__ = _version(__name__) From 776cbea8662790797cd4a4a9c7ca82cdbc381b33 Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Fri, 8 Nov 2024 14:55:05 -0500 Subject: [PATCH 5/6] Get working with ATLAS --- sbn_survey_image_service/services/image.py | 43 ++++++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/sbn_survey_image_service/services/image.py b/sbn_survey_image_service/services/image.py index 6825538..f6f0034 100644 --- a/sbn_survey_image_service/services/image.py +++ b/sbn_survey_image_service/services/image.py @@ -15,6 +15,7 @@ from astropy.coordinates import Angle import astropy.units as u from astropy.io import fits +from astropy.time import Time from astropy.nddata import Cutout2D from astropy.coordinates import SkyCoord, Angle from astropy.wcs import WCS, FITSFixedWarning @@ -25,6 +26,7 @@ from ..models.image import Image from ..config.exceptions import InvalidImageID, ParameterValueError from ..config.env import ENV +from .. import __version__ as sis_version class ImageFormat(Enum): @@ -99,7 +101,7 @@ def normalize(self) -> None: # Dec -90 to 90 self.dec = min(max(self.dec, -90), 90) - def cutout(self, url: str) -> str: + def cutout(self, url: str, wcs_ext: int, data_ext: int, meta: dict={}) -> str: """Generate a cutout from URL. @@ -108,6 +110,15 @@ def cutout(self, url: str) -> str: url : str The URL to the full-size image. + wcs_ext : int + The FITS HDU extension with the WCS. + + data_ext : int + The FITS HDU extension with the data to cutout. + + meta : dict, optional + Optional metadata to add to the FITS header. + Returns ------- @@ -139,25 +150,33 @@ def cutout(self, url: str) -> str: "fsspec_kwargs": {"block_size": 1024 * 512, "cache_type": "bytes"}, } with fits.open(url, **options) as data: - i: int = 0 - - header: fits.Header = copy(data[i].header) + wcs_header: fits.Header = copy(data[wcs_ext].header) wcs: WCS with warnings.catch_warnings(): warnings.simplefilter( "ignore", (fits.verify.VerifyWarning, FITSFixedWarning) ) - wcs = WCS(header) + wcs = WCS(wcs_header) cutout: Cutout2D = Cutout2D( - data[i].section, self.coords, self.size, wcs=wcs + data[data_ext].section, self.coords, self.size, wcs=wcs ) + header: fits.Header = copy(data[data_ext].header) header.update(cutout.wcs.to_header()) + header.add_comment("Cutout generated by the SBN Survey Image Service") + header.add_comment("NASA Planetary Data System Small-Bodies Node") + header.add_comment(f"version {sis_version}") + header.add_comment(f"date {Time.now().iso}") + header["sis-ra"] = self.ra, "cutout center RA (deg)" + header["sis-dec"] = self.dec, "cutout center Dec (deg)" + header["sis-size"] = str(self.size), "cutout size" + for k, v in meta.items(): + header[k] = v result.append(fits.PrimaryHDU(cutout.data, header)) - result.writeto(fits_image_path) + result.writeto(fits_image_path, output_verify="silentfix") os.chmod(fits_image_path, 33204) @@ -286,8 +305,16 @@ def image_query( os.path.basename(im.image_url))[0] download_filename += filename_suffix(cutout_spec, format) + # ATLAS data and WCS are found in the first extension + wcs_ext: int = 0 + data_ext: int = 0 + if ":gbo.ast.atlas.survey" in im.collection: + wcs_ext = 1 + data_ext = 1 + # generate the cutout, as needed - fits_image_path: str = cutout_spec.cutout(im.image_url) + meta = {"sis-lid": obs_id} + fits_image_path: str = cutout_spec.cutout(im.image_url, wcs_ext, data_ext, meta=meta) # FITS format? done! if format == ImageFormat.FITS: From 28030c2cb6cfec1dcd18765b05076a8b1ec80719 Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Fri, 8 Nov 2024 15:00:23 -0500 Subject: [PATCH 6/6] Reformat with black --- _precommit_hook | 3 +-- pyproject.toml | 2 +- sbn_survey_image_service/app.py | 3 +-- sbn_survey_image_service/data/add.py | 11 ++++----- .../data/test/generate.py | 24 +++++++++---------- .../models/test/test_models.py | 19 ++++----------- sbn_survey_image_service/scripts/sbnsis.py | 6 ++--- sbn_survey_image_service/services/image.py | 20 ++++++++-------- sbn_survey_image_service/services/label.py | 3 +-- .../test/test_services.py | 17 ++++--------- 10 files changed, 41 insertions(+), 67 deletions(-) diff --git a/_precommit_hook b/_precommit_hook index 0aa7e56..592a08c 100755 --- a/_precommit_hook +++ b/_precommit_hook @@ -14,6 +14,5 @@ if [ $DONT_FORMAT_ON_CODE_COMMIT ]; then """ else # Auto-format all python scripts - .venv/bin/autopep8 -ir sbn_survey_image_service/** - .venv/bin/autopep8 -ir tests/** + .venv/bin/black sbn_survey_image_service/** fi diff --git a/pyproject.toml b/pyproject.toml index bddc9ee..e1fd5da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ [project.optional-dependencies] recommended = ["psycopg2-binary>=2.8"] -dev = ["autopep8", "mypy", "pycodestyle"] +dev = ["black", "mypy", "pycodestyle"] test = ["pytest>=7.0", "pytest-cov>=3.0"] docs = ["sphinx", "sphinx-automodapi", "numpydoc"] diff --git a/sbn_survey_image_service/app.py b/sbn_survey_image_service/app.py index ef8fcbd..a349cb7 100755 --- a/sbn_survey_image_service/app.py +++ b/sbn_survey_image_service/app.py @@ -69,5 +69,4 @@ def handle_other_error(error: Exception): # for development logger.info("Running " + ENV.APP_NAME) logger.info(application.url_map) - app.run("sbn_survey_image_service.app:app", - host=ENV.API_HOST, port=ENV.API_PORT) + app.run("sbn_survey_image_service.app:app", host=ENV.API_HOST, port=ENV.API_PORT) diff --git a/sbn_survey_image_service/data/add.py b/sbn_survey_image_service/data/add.py index 4fc552b..ce13f47 100644 --- a/sbn_survey_image_service/data/add.py +++ b/sbn_survey_image_service/data/add.py @@ -33,7 +33,7 @@ def _remove_prefix(s: str, prefix: str): """If ``s`` starts with ``prefix`` remove it.""" if s.startswith(prefix): - return s[len(prefix):] + return s[len(prefix) :] else: return s @@ -162,8 +162,7 @@ def pds4_image(label_path: str) -> Image: "/Internal_Reference/[reference_type='is_instrument']/../name" ).text.split() ), - target=label.find( - "Observation_Area/Target_Identification/name").text, + target=label.find("Observation_Area/Target_Identification/name").text, calibration_level=PDS4CalibrationLevel[ label.find( "Observation_Area/Primary_Result_Summary/processing_level" @@ -300,8 +299,7 @@ def add_directory( for filename in filenames: if os.path.splitext(filename)[1].lower() in extensions: n_files += 1 - n_added += add_label(os.path.join(dirpath, - filename), session, **kwargs) + n_added += add_label(os.path.join(dirpath, filename), session, **kwargs) if not recursive: break @@ -358,8 +356,7 @@ def __main__() -> None: logger.setLevel(logging.DEBUG if args.v else logging.INFO) # options to pass on to add_* functions: - kwargs = dict(base_url=args.base_url, - strip_leading=args.strip_leading.rstrip("/")) + kwargs = dict(base_url=args.base_url, strip_leading=args.strip_leading.rstrip("/")) session: Session with data_provider_session() as session: if args.create: diff --git a/sbn_survey_image_service/data/test/generate.py b/sbn_survey_image_service/data/test/generate.py index 370e017..55eabc4 100644 --- a/sbn_survey_image_service/data/test/generate.py +++ b/sbn_survey_image_service/data/test/generate.py @@ -93,8 +93,7 @@ def create_data(session, path): logger.info("Creating ~400 images and labels.") centers: np.ndarray = np.degrees(spherical_distribution(400)) image_size: int = 300 - pixel_size: float = np.degrees( - np.sqrt(4 * np.pi / len(centers))) / image_size / 10 + pixel_size: float = np.degrees(np.sqrt(4 * np.pi / len(centers))) / image_size / 10 xy: np.ndarray = np.mgrid[:image_size, :image_size][::-1] w: WCS = WCS() @@ -132,8 +131,7 @@ def create_data(session, path): continue hdu: fits.HDUList = fits.HDUList() - hdu.append(fits.PrimaryHDU( - data.astype(np.int32), header=w.to_header())) + hdu.append(fits.PrimaryHDU(data.astype(np.int32), header=w.to_header())) hdu.writeto(image_path, overwrite=True) outf: io.IOBase with open(label_path, "w") as outf: @@ -183,8 +181,11 @@ def create_tables() -> None: def delete_data(session) -> None: """Delete test data from database.""" - (session.query(Image).filter(Image.collection == - "urn:nasa:pds:survey:test-collection").delete()) + ( + session.query(Image) + .filter(Image.collection == "urn:nasa:pds:survey:test-collection") + .delete() + ) def exists(session) -> bool: @@ -196,8 +197,9 @@ def exists(session) -> bool: try: results: Any = ( - session.query(Image).filter( - Image.collection == "urn:nasa:pds:survey:test-collection").all() + session.query(Image) + .filter(Image.collection == "urn:nasa:pds:survey:test-collection") + .all() ) except OperationalError: return False @@ -228,8 +230,7 @@ def _parse_args() -> argparse.Namespace: default=ENV.TEST_DATA_PATH, help="directory to which to save test data files", ) - parser.add_argument("--add", action="store_true", - help="add/create test data set") + parser.add_argument("--add", action="store_true", help="add/create test data set") parser.add_argument( "--exists", action="store_true", @@ -266,8 +267,7 @@ def _main() -> None: create_data(session, args.path) elif args.delete: delete_data(session) - logger.info( - "Database cleaned, but test files must be removed manually.") + logger.info("Database cleaned, but test files must be removed manually.") elif args.exists: if exists(session): print("Test data set appears to be valid.") diff --git a/sbn_survey_image_service/models/test/test_models.py b/sbn_survey_image_service/models/test/test_models.py index e099526..cc06b5e 100644 --- a/sbn_survey_image_service/models/test/test_models.py +++ b/sbn_survey_image_service/models/test/test_models.py @@ -7,20 +7,9 @@ class TestImage: """Test Image object odds and ends.""" def test_repr(self): - im: Image = Image( - obs_id='asdf', - image_url='fdsa', - label_url='jkl;' - ) - assert ( - repr(im) - == "Image(obs_id='asdf', image_url='fdsa', label_url='jkl;')" - ) + im: Image = Image(obs_id="asdf", image_url="fdsa", label_url="jkl;") + assert repr(im) == "Image(obs_id='asdf', image_url='fdsa', label_url='jkl;')" def test_str(self): - im: Image = Image( - obs_id='asdf', - image_url='fdsa', - label_url='jkl;' - ) - assert str(im) == '' + im: Image = Image(obs_id="asdf", image_url="fdsa", label_url="jkl;") + assert str(im) == "" diff --git a/sbn_survey_image_service/scripts/sbnsis.py b/sbn_survey_image_service/scripts/sbnsis.py index c989344..96f1ac4 100644 --- a/sbn_survey_image_service/scripts/sbnsis.py +++ b/sbn_survey_image_service/scripts/sbnsis.py @@ -163,8 +163,7 @@ def restart(self) -> None: os.kill(ppid, signal.SIGWINCH) ellipsis(10) - print_color(" - Stopping old service parent process", - end="", flush=True) + print_color(" - Stopping old service parent process", end="", flush=True) os.kill(ppid, signal.SIGQUIT) ellipsis(1) @@ -243,8 +242,7 @@ def env_file(self) -> None: print_color("Wrote new .env file.") def argument_parser(self) -> ArgumentParser: - parser: ArgumentParser = ArgumentParser( - description="SBN Survey Image Service") + parser: ArgumentParser = ArgumentParser(description="SBN Survey Image Service") subparsers = parser.add_subparsers(help="sub-command help") # start ######### diff --git a/sbn_survey_image_service/services/image.py b/sbn_survey_image_service/services/image.py index f6f0034..0281967 100644 --- a/sbn_survey_image_service/services/image.py +++ b/sbn_survey_image_service/services/image.py @@ -69,8 +69,7 @@ def __init__(self, ra: float | None, dec: float | None, size: str | Angle | None self.normalize() self.size: Angle = ( - self.MINUMUM_SIZE if size is None else max( - self.MINUMUM_SIZE, Angle(size)) + self.MINUMUM_SIZE if size is None else max(self.MINUMUM_SIZE, Angle(size)) ) def __str__(self) -> str: @@ -101,7 +100,7 @@ def normalize(self) -> None: # Dec -90 to 90 self.dec = min(max(self.dec, -90), 90) - def cutout(self, url: str, wcs_ext: int, data_ext: int, meta: dict={}) -> str: + def cutout(self, url: str, wcs_ext: int, data_ext: int, meta: dict = {}) -> str: """Generate a cutout from URL. @@ -207,7 +206,7 @@ def filename_suffix(cutout_spec: CutoutSpec, format: ImageFormat) -> str: suffix: str = "" if not cutout_spec.full_size: # attachment file name is based on coordinates and size - suffix = f'_{cutout_spec.ra:.5f}{cutout_spec.dec:+.5f}_{cutout_spec.size}' + suffix = f"_{cutout_spec.ra:.5f}{cutout_spec.dec:+.5f}_{cutout_spec.size}" return f"{suffix}.{format.extension}" @@ -286,8 +285,7 @@ def image_query( try: format = ImageFormat(format) except ValueError: - raise ParameterValueError( - "image_query format must be fits, png, or jpeg.") + raise ParameterValueError("image_query format must be fits, png, or jpeg.") im: Image session: Session @@ -301,8 +299,7 @@ def image_query( session.expunge(im) # create attachment file name - download_filename: str = os.path.splitext( - os.path.basename(im.image_url))[0] + download_filename: str = os.path.splitext(os.path.basename(im.image_url))[0] download_filename += filename_suffix(cutout_spec, format) # ATLAS data and WCS are found in the first extension @@ -314,7 +311,9 @@ def image_query( # generate the cutout, as needed meta = {"sis-lid": obs_id} - fits_image_path: str = cutout_spec.cutout(im.image_url, wcs_ext, data_ext, meta=meta) + fits_image_path: str = cutout_spec.cutout( + im.image_url, wcs_ext, data_ext, meta=meta + ) # FITS format? done! if format == ImageFormat.FITS: @@ -322,7 +321,8 @@ def image_query( # formulate the final image file name image_path = generate_cache_filename( - im.image_url, str(cutout_spec), format.extension) + im.image_url, str(cutout_spec), format.extension + ) # was this file already generated? serve it! if os.path.exists(image_path): diff --git a/sbn_survey_image_service/services/label.py b/sbn_survey_image_service/services/label.py index 3d8ba0c..6024e6f 100644 --- a/sbn_survey_image_service/services/label.py +++ b/sbn_survey_image_service/services/label.py @@ -21,8 +21,7 @@ def label_query(obs_id: str) -> Tuple[str, str]: exc: Exception try: label_url: str = ( - session.query(Image.label_url).filter( - Image.obs_id == obs_id).one()[0] + session.query(Image.label_url).filter(Image.obs_id == obs_id).one()[0] ) except NoResultFound as exc: raise InvalidImageID("Image ID not found in database.") from exc diff --git a/sbn_survey_image_service/test/test_services.py b/sbn_survey_image_service/test/test_services.py index eb2cfde..a705456 100644 --- a/sbn_survey_image_service/test/test_services.py +++ b/sbn_survey_image_service/test/test_services.py @@ -32,8 +32,7 @@ def test_label_query(): image_path, download_filename = label_query( "urn:nasa:pds:survey:test-collection:test-000039" ) - assert image_path == os.path.join( - "file://", ENV.TEST_DATA_PATH, "test-000039.xml") + assert image_path == os.path.join("file://", ENV.TEST_DATA_PATH, "test-000039.xml") def test_label_query_fail(): @@ -67,8 +66,7 @@ def test_image_query_full_frame_jpg(): ) # should return a file in the cache directory - assert os.path.dirname(image_path) == os.path.abspath( - ENV.SBNSIS_CUTOUT_CACHE) + assert os.path.dirname(image_path) == os.path.abspath(ENV.SBNSIS_CUTOUT_CACHE) assert image_path == expected_path assert download_filename == "test-000023.jpeg" @@ -87,8 +85,7 @@ def test_image_query_full_frame_png(): ) # should return a file in the cache directory - assert os.path.dirname(image_path) == os.path.abspath( - ENV.SBNSIS_CUTOUT_CACHE) + assert os.path.dirname(image_path) == os.path.abspath(ENV.SBNSIS_CUTOUT_CACHE) assert image_path == expected_path assert download_filename == "test-000023.png" @@ -117,13 +114,9 @@ def test_image_query_cutout(): ) # should return fits file in cache directory - assert os.path.dirname(image_path) == os.path.abspath( - ENV.SBNSIS_CUTOUT_CACHE) + assert os.path.dirname(image_path) == os.path.abspath(ENV.SBNSIS_CUTOUT_CACHE) assert image_path == expected_path - assert ( - download_filename - == f'test-000102_{+ra:.5f}{+dec:.5f}_{size}.fits' - ) + assert download_filename == f"test-000102_{+ra:.5f}{+dec:.5f}_{size}.fits" # inspect file, value should be -25 at the center im: np.ndarray = fits.getdata(image_path)