diff --git a/_activate b/_activate
index 5d7dc1f..c8423a3 100644
--- a/_activate
+++ b/_activate
@@ -1,7 +1,7 @@
#!/usr/bin/env false
if [ ! -f $PWD/.env ]; then
- echo -e "No .env file found. Generate with '_sbnsis env' then edit."
+ echo -e "No .env file found. Generate with 'sbnsis env' then edit."
return 1
fi
source .env
diff --git a/_add_atlas b/_add_atlas
new file mode 100755
index 0000000..d01386c
--- /dev/null
+++ b/_add_atlas
@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""
+Identify new ATLAS data and add to the SBN SIS database.
+"""
+
+import os
+import sys
+import shlex
+import logging
+import sqlite3
+import argparse
+from glob import glob
+import logging.handlers
+from packaging.version import Version
+
+from astropy.time import Time
+import astropy.units as u
+import pds4_tools
+
+from sbn_survey_image_service.data.add import add_label
+from sbn_survey_image_service.services.database_provider import data_provider_session
+
+
+class LabelError(Exception):
+ pass
+
+
+def get_logger():
+ return logging.getLogger("SBNSIS/Add ATLAS")
+
+
+def setup_logger(args):
+ logger = get_logger()
+
+ if len(logger.handlers) > 0:
+ # already set up
+ return logger
+
+ if not os.path.exists(os.path.dirname(args.log)):
+ os.makedirs(os.path.dirname(args.log), exist_ok=True)
+
+ logger.setLevel(logging.DEBUG)
+
+ formatter = logging.Formatter("%(levelname)s:%(name)s:%(asctime)s: %(message)s")
+
+ handler = logging.StreamHandler(sys.stderr)
+ handler.setLevel(logging.DEBUG if args.verbose else logging.ERROR)
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ handler = logging.FileHandler(args.log)
+ handler.setLevel(logging.DEBUG if args.verbose else logging.INFO)
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ logger.info("%s", " ".join([shlex.quote(s) for s in sys.argv]))
+
+ return logger
+
+
+def collection_version(collection) -> Version:
+ """Get the collection version."""
+ is_collection = (
+ collection.label.find("Identification_Area/product_class").text
+ == "Product_Collection"
+ )
+ vid = collection.label.find("Identification_Area/version_id")
+ if not is_collection or vid is None:
+ raise LabelError("This does not appear to be a valid PDS4 label.")
+ return Version(vid.text)
+
+
+def get_lidvid(filename):
+ """Return the LIDVID and data file name."""
+ product = pds4_tools.read(filename, quiet=True, lazy_load=True)
+ lid = product.label.find("Identification_Area/logical_identifier").text
+ vid = product.label.find("Identification_Area/version_id").text
+ return "::".join((lid, vid))
+
+
+def get_image_labels(collection, data_directory) -> Version:
+ """Get the file inventory of image files to ingest.
+
+ The label file names for all LIDVIDs ending with ".fits" in the collection
+ inventory will be returned.
+
+ Candidate labels are collected from xml files within `directory`.
+
+ """
+
+ logger = get_logger()
+ files = {}
+ count = 0
+ for fn in glob(f"{data_directory}/*xml"):
+ if not fn.endswith(".fits.xml"):
+ continue
+ files[get_lidvid(fn)] = fn
+ count += 1
+ if (count % 100) == 0:
+ logger.debug("%d files read", count)
+ logger.debug("%d files read", count)
+
+ image_files = []
+ for lidvid in collection[0].data["LIDVID_LID"]:
+ lid = lidvid.split("::")[0]
+ if not lid.endswith(".fits"):
+ continue
+ if lidvid not in files:
+ raise LabelError(f"{lidvid} not found in {data_directory}")
+ image_files.append(files[lidvid])
+ return image_files
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "database", type=os.path.normpath, help="ATLAS-PDS processing database"
+)
+mutex = parser.add_mutually_exclusive_group()
+mutex.add_argument(
+ "--since-date", type=Time, help="harvest metadata validated since this date"
+)
+mutex.add_argument(
+ "--since",
+ type=int,
+ help="harvest metadata validated in the past SINCE hours (default: 24)",
+)
+parser.add_argument(
+ "--log", default="./logging/add-atlas.log", help="log messages to this file"
+)
+parser.add_argument(
+ "--verbose", "-v", action="store_true", help="log debugging messages"
+)
+args = parser.parse_args()
+
+logger = setup_logger(args)
+
+# setup database
+try:
+ db = sqlite3.connect(f"file:{args.database}?mode=ro", uri=True)
+ db.row_factory = sqlite3.Row
+except Exception as exc:
+ logger.error("Could not connect to database %s", args.database)
+ raise exc
+
+logger.info("Connected to database %s", args.database)
+
+if args.since_date:
+ date = args.since_date
+else:
+ date = Time.now() - args.since * u.hr
+logger.info("Checking for collections validated since %s", date.iso)
+
+# check for new collections
+cursor = db.execute(
+ "SELECT * FROM nn WHERE current_status = 'validated' AND recorded_at > ?",
+ (date.unix,),
+)
+results = cursor.fetchall()
+
+if len(results) == 0:
+ logger.info("No new data collections found.")
+else:
+ with data_provider_session() as session:
+ for row in results:
+ collections = [
+ pds4_tools.read(fn, quiet=True)
+ for fn in glob(f"/n/{row['location']}/collection_{row['nn']}*.xml")
+ ]
+ versions = [collection_version(label) for label in collections]
+ latest = collections[versions.index(max(versions))]
+ lid = latest.label.find("Identification_Area/logical_identifier").text
+ vid = latest.label.find("Identification_Area/version_id").text
+ logger.info("Found collection %s::%s", lid, vid)
+
+ data_directory = f"/n/{row['location']}/data"
+ logger.debug(
+ "Inspecting directory %s for image products",
+ data_directory,
+ )
+ files = get_image_labels(latest, data_directory)
+ logger.info("%d image products to add", len(files))
+
+ count = 0
+ errored = 0
+ for label in files:
+ try:
+ count += add_label(label, session)
+ except Exception as exc:
+ errored += 1
+ logger.info(
+ "%d files added, %d files already in the database, %d files errored.",
+ count,
+ len(files) - count - errored,
+ errored,
+ )
+logger.info("Finished.")
diff --git a/_develop_apis b/_develop_apis
index d2b5342..58ebf1b 100755
--- a/_develop_apis
+++ b/_develop_apis
@@ -1,2 +1,2 @@
#! /bin/bash
-_sbnsis start --dev
+sbnsis start --dev
diff --git a/_initial_setup.sh b/_initial_setup.sh
index 2cb760e..745aedb 100755
--- a/_initial_setup.sh
+++ b/_initial_setup.sh
@@ -75,7 +75,7 @@ if [ ! -f $PWD/.env ]; then
To create a .env file:
source .venv/bin/activate
- _sbnsis env
+ sbnsis env
Then edit .env
${reset_color}"""
diff --git a/_sbnsis b/_sbnsis
deleted file mode 100755
index 225dd83..0000000
--- a/_sbnsis
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python3
-from sbn_survey_image_service.scripts.sbnsis import SBNSISService
-SBNSISService()
diff --git a/pyproject.toml b/pyproject.toml
index c0ed28f..5c78c73 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,23 +2,24 @@
name = "sbn-survey-image-service"
description = "Serves images and cutouts via REST API."
readme = "readme.md"
-authors = [
- { name = "Michael S. P. Kelley", email = "msk@astro.umd.edu" }
-]
+authors = [{ name = "Michael S. P. Kelley", email = "msk@astro.umd.edu" }]
license = { text = "BSD 3-Clause License" }
+dynamic = ["version"]
+requires-python = ">= 3.10"
+
dependencies = [
- "Flask>=3.0",
- "Flask-Cors>=4.0",
- "gunicorn>=21",
- "connexion>=3.0",
- "swagger-ui-bundle>1.0",
"astropy>=6.0",
+ "connexion[flask,swagger-ui,uvicorn]~=3.0",
+ "gunicorn~=21.2",
"pds4_tools==1.3",
- "SQLAlchemy>=2.0",
- "python-dotenv>1.0",
"pytest-remotedata>=0.4",
+ "python-dotenv~=1.0",
+ "SQLAlchemy>=2.0",
]
-dynamic = ["version"]
+
+[project.optional-dependencies]
+recommended = ["psycopg2-binary>=2.8"]
+dev = ["autopep8", "mypy", "pycodestyle", "pytest>=7.0", "pytest-cov>=3.0"]
[project.urls]
homepage = "https://github.com/Small-Bodies-Node/sbn-survey-image-service"
@@ -28,20 +29,13 @@ requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
build-backend = 'setuptools.build_meta'
[tool.setuptools_scm]
+write_to = "sbn_survey_image_service/_version.py"
[tool.setuptools]
zip-safe = false
[tool.setuptools.packages.find]
-[project.optional-dependencies]
-recommended = [
- "psycopg2-binary>=2.8",
-]
-dev = [
- "autopep8",
- "mypy",
- "pycodestyle",
- "pytest>=7.0",
- "pytest-cov>=3.0",
-]
\ No newline at end of file
+[project.scripts]
+sbnsis = "sbn_survey_image_service.scripts.sbnsis:__main__"
+sbnsis-add = "sbn_survey_image_service.data.add:__main__"
diff --git a/readme.md b/readme.md
index 4aa3405..ffc4c2c 100644
--- a/readme.md
+++ b/readme.md
@@ -34,7 +34,7 @@ This repo has code for:
- Running the API
- Testing
-Most day-to-day tasks can be accomplished with the `_sbnsis` command.
+Most day-to-day tasks can be accomplished with the `sbnsis` command.
The following steps are needed to set up the code base:
@@ -55,7 +55,7 @@ The following steps are needed to set up the code base:
- Create and activate a python virtual environment.
- To use a specific Python interpreter, set the PYTHON environment variable: `PYTHON=/path/to/python3 bash _install_setup.sh`
- Install dependencies, including `fitscut`, to the virtual env.
-- Create a new environment variable file and edit to suit your needs: `_sbnsis env`.
+- Create a new environment variable file and edit to suit your needs: `sbnsis env`.
- Optionally test your set up:
- Be aware that the testing suite will use the database parameters specified in the `.env` file.
- The database user must have write permissions for testing.
@@ -69,17 +69,17 @@ The following steps are needed to set up the code base:
### Adding archival data
-The `sbn_survey_image_service.data.add` sub-module is used to add image metadata to the database. It scans PDS3 or PDS4 labels, and saves to the database data product metadata and URLs to the label and image data. The sub-module may be run as a command-line script `python3 -m sbn_survey_image_service.data.add`. The script will automatically create the database in case it does not exist. For example, to search a NEAT survey directory for PDS4 image labels and data, and to form URLs with which the data may be retrieved:
+The `sbn_survey_image_service.data.add` sub-module is used to add image metadata to the database. It scans PDS3 or PDS4 labels, and saves to the database data product metadata and URLs to the label and image data. The sub-module provides a command-line script `sbnsis-add`. The script will automatically create the database in case it does not exist. For example, to search a NEAT survey directory for PDS4 image labels and data, and to form URLs with which the data may be retrieved:
```
-python3 -m sbn_survey_image_service.data.add -r \
+sbnsis-add -r \
/path/to/gbo.ast.neat.survey/data_geodss/g19960417/obsdata
```
-The previous example is for a survey accessible via the local file system. As an alternative, data may be served to the image service via HTTP(S). In this case, the `add` script must still be run on locally accessible labels, but an appropriate URL may be formed using the `--base-url` and `--strip-leading` parameters:
+The previous example is for a survey accessible via the local file system. As an alternative, data may be served to the image service via HTTP(S). In this case, the `sbnsis-add` script must still be run on locally accessible labels, but an appropriate URL may be formed using the `--base-url` and `--strip-leading` parameters:
```
-python3 -m sbn_survey_image_service.data.add -r \
+sbnsis-add -r \
/path/to/gbo.ast.neat.survey/data_geodss/g19960417/obsdata \
--base-url=https://sbnarchive.psi.edu/pds4/surveys \
--strip-leading=/path/to/
@@ -87,7 +87,7 @@ python3 -m sbn_survey_image_service.data.add -r \
For a summary of command-line parameters, use the `--help` option.
-Due to survey-to-survey label differences, it is unlikely that the script will work with a previously untested data source. Edit the appropriate functions in `sbn_survey_image_service/data/add.py`, either `pds3_image` or `pds4_image`. For example, the NEAT survey PDS4 data set v1.0 does not have pixel scale in the label, so we have hard coded it into the `pds4_image` function.
+Due to survey-to-survey label differences, it is unlikely that the script will work with a previously untested data source. Edit the appropriate functions in `sbn_survey_image_service/data/add.py`, e.g., `pds4_image()`. For example, the NEAT survey PDS4 data set v1.0 does not have pixel scale in the label, so we have hard coded it into the `pds4_image` function.
It is assumed that survey images are FITS-compatible with a World Coordinate System defined for a standard sky reference frame (ICRS). The cutout service uses the FITS header, not the PDS labels, to define the sub-frame. This is a limitation from using `fitscut`.
@@ -97,11 +97,11 @@ Whether running in development or deployment modes, the Swagger documentation is
### Development
-If you have `nodemon` globally installed, then you can develop your api code and have it automatically update on changes by running `_sbnsis start --dev_`. Otherwise, just run `python -m sbn_survey_image_service.api.app`.
+If you have `nodemon` globally installed, then you can develop your api code and have it automatically update on changes by running `sbnsis start --dev_`. Otherwise, just run `python -m sbn_survey_image_service.app`.
### Deployment
-The `_sbnsis` takes the arguments `start|stop|status|restart` to launch the app as a background process with the gunicorn WSGI server for production serving. The number of workers is controlled with the env variable `LIVE_GUNICORN_INSTANCES`. If you have trouble getting gunicorn to work, running in non-daemon mode may help with debugging: `_sbnsis start --no-daemon`.
+The `sbnsis` takes the arguments `start|stop|status|restart` to launch the app as a background process with the gunicorn WSGI server for production serving. The number of workers is controlled with the env variable `LIVE_GUNICORN_INSTANCES`. If you have trouble getting gunicorn to work, running in non-daemon mode may help with debugging: `sbnsis start --no-daemon`.
It is recommended that you make the gunicorn-powered server accesible to the outside world by proxy-passing requests through an https-enabled web server like apache.
diff --git a/sbn_survey_image_service/__init__.py b/sbn_survey_image_service/__init__.py
index f2ae0a8..22a019e 100644
--- a/sbn_survey_image_service/__init__.py
+++ b/sbn_survey_image_service/__init__.py
@@ -5,8 +5,8 @@
from importlib.metadata import version as _version, PackageNotFoundError
# make cache directory set umask
-from . import exceptions
-from . import env
+from .config import exceptions
+from .config import env
from . import services
from . import models
diff --git a/sbn_survey_image_service/_version.py b/sbn_survey_image_service/_version.py
new file mode 100644
index 0000000..f20da57
--- /dev/null
+++ b/sbn_survey_image_service/_version.py
@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+ from typing import Tuple, Union
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+ VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.2.0.dev12+gf2ddedd.d20240821'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev12', 'gf2ddedd.d20240821')
diff --git a/sbn_survey_image_service/api/app.py b/sbn_survey_image_service/api/app.py
deleted file mode 100755
index 54eff57..0000000
--- a/sbn_survey_image_service/api/app.py
+++ /dev/null
@@ -1,156 +0,0 @@
-# Licensed under a 3-clause BSD style license - see LICENSE.rst
-"""
-Entry point to Flask-Connexion API
-"""
-
-import os
-import uuid
-import json
-import logging
-from typing import Any, Dict, List, Optional, Union
-from connexion import FlaskApp
-from flask import send_file, Response
-from flask_cors import CORS
-from ..env import ENV
-from ..services import (label_query,
- image_query,
- metadata_query,
- metadata_summary,
- database_provider)
-from ..logging import get_logger
-from ..exceptions import SBNSISException
-
-MIME_TYPES = {
- '.xml': 'text/xml',
- '.fit': 'image/fits',
- '.fits': 'image/fits',
- '.fit.fz': 'image/fits',
- '.fits.fz': 'image/fits',
- '.jpeg': 'image/jpeg',
- '.png': 'image/png'
-}
-
-
-def get_image(id: str, ra: Optional[float] = None, dec: Optional[float] = None,
- size: Optional[str] = None, format: str = 'fits',
- download: bool = False) -> Response:
- """Controller for survey image service."""
-
- logger: logging.Logger = get_logger()
- job_id: uuid.UUID = uuid.uuid4()
- logger.info(json.dumps({'job_id': job_id.hex,
- 'job': 'images',
- 'id': id,
- 'ra': ra,
- 'dec': dec,
- 'size': size,
- 'format': format,
- 'download': download}))
-
- filename: str
- download_filename: str
- if format.lower() == 'label':
- filename, download_filename = label_query(id)
- else:
- filename, download_filename = image_query(
- id, ra=ra, dec=dec, size=size, format=format)
-
- mime_type: str = MIME_TYPES.get(
- os.path.splitext(download_filename.lower())[1],
- 'text/plain')
-
- logger.info(json.dumps({
- 'job_id': job_id.hex,
- 'job': 'images',
- 'filename': filename,
- 'download_filename': download_filename,
- 'mime_type': mime_type
- }))
-
- return send_file(filename, mimetype=mime_type,
- as_attachment=download,
- download_name=download_filename)
-
-
-def run_query(collection: Optional[str] = None,
- facility: Optional[str] = None,
- instrument: Optional[str] = None,
- dptype: Optional[str] = None,
- format: str = 'fits',
- maxrec: int = 100,
- offset: int = 0,
- ) -> Dict[str, Union[int, List[dict]]]:
- """Controller for metadata queries."""
-
- logger: logging.Logger = get_logger()
- job_id: uuid.UUID = uuid.uuid4()
- logger.info(json.dumps({'job_id': job_id.hex,
- 'job': 'query',
- 'collection': collection,
- 'facility': facility,
- 'instrument': instrument,
- 'dptype': dptype,
- 'format': format,
- 'maxrec': maxrec,
- 'offset': offset}))
-
- total, results = metadata_query(collection=collection,
- facility=facility,
- instrument=instrument,
- dptype=dptype,
- format=format,
- maxrec=maxrec,
- offset=offset)
-
- return {'total': total,
- 'offset': offset,
- 'count': len(results),
- 'results': results}
-
-
-def get_summary():
- """Controller for summaries."""
-
- logger: logging.Logger = get_logger()
- job_id: uuid.UUID = uuid.uuid4()
- logger.info(json.dumps({'job_id': job_id.hex}))
-
- summary = metadata_summary()
-
- return summary
-
-
-###########################################
-# BEGIN API
-###########################################
-
-app = FlaskApp(__name__, options={})
-app.add_api('openapi.yaml', base_path=ENV.BASE_HREF)
-CORS(app.app)
-application = app.app
-
-
-@ application.teardown_appcontext
-def shutdown_session(exception: Exception = None) -> None:
- """ Boilerplate connexion code """
- database_provider.db_session.remove()
-
-
-@ application.errorhandler(SBNSISException)
-def handle_sbnsis_error(error: Exception):
- """Log errors."""
- get_logger().exception('SBS Survey Image Service error.')
- return str(error), getattr(error, 'code', 500)
-
-
-@ application.errorhandler(Exception)
-def handle_other_error(error: Exception):
- """Log errors."""
- get_logger().exception('An error occurred.')
- return ('Unexpected error. Please report if the problem persists.',
- getattr(error, 'code', 500))
-
-
-if __name__ == '__main__':
- print(application.url_map)
- app.run(port=ENV.API_PORT, use_reloader=False, threaded=False)
diff --git a/sbn_survey_image_service/api/images.py b/sbn_survey_image_service/api/images.py
new file mode 100644
index 0000000..08ff49f
--- /dev/null
+++ b/sbn_survey_image_service/api/images.py
@@ -0,0 +1,73 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
+import os
+import json
+import uuid
+import logging
+
+from flask import send_file, Response
+
+from ..config import MIME_TYPES
+from ..config.logging import get_logger
+from ..services.label import label_query
+from ..services.image import image_query
+
+
+def get_image(
+ id: str,
+ ra: float | None = None,
+ dec: float | None = None,
+ size: str | None = None,
+ format: str = "fits",
+ download: bool = False,
+) -> Response:
+ """Controller for survey image service."""
+
+ logger: logging.Logger = get_logger()
+ job_id: uuid.UUID = uuid.uuid4()
+ logger.info(
+ json.dumps(
+ {
+ "job_id": job_id.hex,
+ "job": "images",
+ "id": id,
+ "ra": ra,
+ "dec": dec,
+ "size": size,
+ "format": format,
+ "download": download,
+ }
+ )
+ )
+
+ filename: str
+ download_filename: str
+ if format.lower() == "label":
+ filename, download_filename = label_query(id)
+ else:
+ filename, download_filename = image_query(
+ id, ra=ra, dec=dec, size=size, format=format
+ )
+
+ mime_type: str = MIME_TYPES.get(
+ os.path.splitext(download_filename.lower())[1], "text/plain"
+ )
+
+ logger.info(
+ json.dumps(
+ {
+ "job_id": job_id.hex,
+ "job": "images",
+ "filename": filename,
+ "download_filename": download_filename,
+ "mime_type": mime_type,
+ }
+ )
+ )
+
+ return send_file(
+ filename,
+ mimetype=mime_type,
+ as_attachment=download,
+ download_name=download_filename,
+ )
diff --git a/sbn_survey_image_service/api/openapi.yaml b/sbn_survey_image_service/api/openapi.yaml
index 581ee0e..0a42aa2 100644
--- a/sbn_survey_image_service/api/openapi.yaml
+++ b/sbn_survey_image_service/api/openapi.yaml
@@ -2,16 +2,18 @@ openapi: 3.0.0
servers:
- url: https://sbnsurveys.astro.umd.edu/api/
info:
- title: SBN Survey Image Service (Working Draft)
- version: "0.2.0"
+ title: SBN Survey Image Service
+ version: {{version}}
description: API for serving images, cutouts, and data labels from survey data provided by the NASA Planetary Data System Small Bodies Node. Note that this service is more closely aligned with IVOA concepts, rather than PDS4 concepts. (PDS4-concept supported queries will be possible in the future with the PDS4 Registry.)
+servers:
+ - url: {{base_href}}
paths:
/images/{id}:
get:
tags:
- Survey images and labels
summary: Get survey image (full-size or sub-frame) or label corresponding to the requested image ID. Image data may be returned in FITS, JPEG, or PNG formats.
- operationId: sbn_survey_image_service.api.app.get_image
+ operationId: sbn_survey_image_service.api.images.get_image
parameters:
- name: id
in: path
@@ -89,7 +91,7 @@ paths:
tags:
- Search survey metadata
summary: Search survey metadata
- operationId: sbn_survey_image_service.api.app.run_query
+ operationId: sbn_survey_image_service.api.query.run_query
parameters:
- name: collection
in: query
@@ -201,7 +203,7 @@ paths:
tags:
- Database summary
summary: Database summary, describing the collections, facilities, instruments, and number of data products.
- operationId: sbn_survey_image_service.api.app.get_summary
+ operationId: sbn_survey_image_service.api.summary.get_summary
responses:
"200":
description: Summary
diff --git a/sbn_survey_image_service/api/query.py b/sbn_survey_image_service/api/query.py
new file mode 100644
index 0000000..d101b34
--- /dev/null
+++ b/sbn_survey_image_service/api/query.py
@@ -0,0 +1,51 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
+import json
+import uuid
+import logging
+from typing import Dict, List
+
+from ..config.logging import get_logger
+from ..services.metadata import metadata_query
+
+
+def run_query(
+ collection: str | None = None,
+ facility: str | None = None,
+ instrument: str | None = None,
+ dptype: str | None = None,
+ format: str = "fits",
+ maxrec: int = 100,
+ offset: int = 0,
+) -> Dict[str, int | List[dict]]:
+ """Controller for metadata queries."""
+
+ logger: logging.Logger = get_logger()
+ job_id: uuid.UUID = uuid.uuid4()
+ logger.info(
+ json.dumps(
+ {
+ "job_id": job_id.hex,
+ "job": "query",
+ "collection": collection,
+ "facility": facility,
+ "instrument": instrument,
+ "dptype": dptype,
+ "format": format,
+ "maxrec": maxrec,
+ "offset": offset,
+ }
+ )
+ )
+
+ total, results = metadata_query(
+ collection=collection,
+ facility=facility,
+ instrument=instrument,
+ dptype=dptype,
+ format=format,
+ maxrec=maxrec,
+ offset=offset,
+ )
+
+ return {"total": total, "offset": offset, "count": len(results), "results": results}
diff --git a/sbn_survey_image_service/api/summary.py b/sbn_survey_image_service/api/summary.py
new file mode 100644
index 0000000..834b9ad
--- /dev/null
+++ b/sbn_survey_image_service/api/summary.py
@@ -0,0 +1,21 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
+import json
+import uuid
+import logging
+from typing import List
+
+from ..config.logging import get_logger
+from ..services.metadata import metadata_summary
+
+
+def get_summary() -> List[dict]:
+ """Controller for summaries."""
+
+ logger: logging.Logger = get_logger()
+ job_id: uuid.UUID = uuid.uuid4()
+ logger.info(json.dumps({"job_id": job_id.hex}))
+
+ summary: List[dict] = metadata_summary()
+
+ return summary
diff --git a/sbn_survey_image_service/app.py b/sbn_survey_image_service/app.py
new file mode 100755
index 0000000..a349cb7
--- /dev/null
+++ b/sbn_survey_image_service/app.py
@@ -0,0 +1,72 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+"""
+Entry point to Flask-Connexion API
+"""
+
+import logging
+
+import connexion
+from connexion.middleware import MiddlewarePosition
+from starlette.middleware.cors import CORSMiddleware
+
+from . import __version__ as version
+from .config.logging import get_logger
+from .config.env import ENV
+from .config.exceptions import SBNSISException
+from .services.database_provider import db_session
+
+logger: logging.Logger = get_logger()
+app = connexion.FlaskApp(__name__, specification_dir="api/")
+
+app.add_middleware(
+ CORSMiddleware,
+ position=MiddlewarePosition.BEFORE_EXCEPTION,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+app.add_api(
+ "openapi.yaml",
+ arguments={
+ "version": version,
+ "base_href": ENV.BASE_HREF,
+ },
+)
+application = app.app
+
+
+@application.teardown_appcontext
+def shutdown_db_session(exception: Exception = None) -> None:
+ db_session.remove()
+
+
+@application.errorhandler(SBNSISException)
+def handle_sbnsis_error(error: Exception):
+ """Log errors.
+
+ The HTTP status code is based on the exception, or 500 if it is not defined.
+
+ """
+
+ get_logger().exception("SBS Survey Image Service error.")
+ return str(error), getattr(error, "code", 500)
+
+
+@application.errorhandler(Exception)
+def handle_other_error(error: Exception):
+ """Log errors."""
+
+ get_logger().exception("An error occurred.")
+ return (
+ "Unexpected error. Please report if the problem persists.",
+ getattr(error, "code", 500),
+ )
+
+
+if __name__ == "__main__":
+ # for development
+ logger.info("Running " + ENV.APP_NAME)
+ logger.info(application.url_map)
+ app.run("sbn_survey_image_service.app:app", host=ENV.API_HOST, port=ENV.API_PORT)
diff --git a/sbn_survey_image_service/config/__init__.py b/sbn_survey_image_service/config/__init__.py
new file mode 100644
index 0000000..81a0337
--- /dev/null
+++ b/sbn_survey_image_service/config/__init__.py
@@ -0,0 +1,8 @@
+MIME_TYPES = {
+ ".xml": "text/xml",
+ ".fit": "image/fits",
+ ".fits": "image/fits",
+ ".fz": "image/fits",
+ ".jpeg": "image/jpeg",
+ ".png": "image/png",
+}
diff --git a/sbn_survey_image_service/env.py b/sbn_survey_image_service/config/env.py
similarity index 92%
rename from sbn_survey_image_service/env.py
rename to sbn_survey_image_service/config/env.py
index 01ddb48..760914e 100644
--- a/sbn_survey_image_service/env.py
+++ b/sbn_survey_image_service/config/env.py
@@ -6,12 +6,13 @@
import multiprocessing
from typing import List, Union
from dotenv import load_dotenv, find_dotenv
+
load_dotenv(find_dotenv(), override=True, verbose=True)
__all__: List[str] = ["ENV", "env_example"]
-class SBNSISEnvironment():
+class SBNSISEnvironment:
"""Defines environment variables and their defaults.
To add new variables, edit this class and `env_example`.
@@ -19,7 +20,7 @@ class SBNSISEnvironment():
"""
# Logging
- SBNSIS_LOG_FILE: str = os.path.abspath('./logging/sbnsis.log')
+ SBNSIS_LOG_FILE: str = os.path.abspath("./logging/sbnsis.log")
# Data parameters
TEST_DATA_PATH: str = os.path.abspath("./data/test")
@@ -36,6 +37,7 @@ class SBNSISEnvironment():
# Gunicorn parameters
LIVE_GUNICORN_INSTANCES: int = -1
APP_NAME: str = "sbnsis-service"
+ API_HOST: str = "0.0.0.0"
API_PORT: int = 5000
BASE_HREF: str = "/"
IS_DAEMON: str = "TRUE"
@@ -59,7 +61,8 @@ def __init__(self):
ENV: SBNSISEnvironment = SBNSISEnvironment()
-env_example: str = f"""
+env_example: str = (
+ f"""
# sbnsis configuration
################
@@ -94,6 +97,7 @@ def __init__(self):
# API CONFIG
APP_NAME={SBNSISEnvironment.APP_NAME}
+API_HOST={SBNSISEnvironment.API_HOST}
API_PORT={SBNSISEnvironment.API_PORT}
BASE_HREF={SBNSISEnvironment.BASE_HREF}
@@ -111,9 +115,10 @@ def __init__(self):
TEST_DATA_PATH={SBNSISEnvironment.TEST_DATA_PATH}
# log file
-# _sbnsis will rotate any files matching "*.log" in the ./logging directory
+# sbnsis will rotate any files matching "*.log" in the ./logging directory
SBNSIS_LOG_FILE={SBNSISEnvironment.SBNSIS_LOG_FILE}
""".strip()
+)
# Debugging block
diff --git a/sbn_survey_image_service/exceptions.py b/sbn_survey_image_service/config/exceptions.py
similarity index 100%
rename from sbn_survey_image_service/exceptions.py
rename to sbn_survey_image_service/config/exceptions.py
diff --git a/sbn_survey_image_service/logging.py b/sbn_survey_image_service/config/logging.py
similarity index 80%
rename from sbn_survey_image_service/logging.py
rename to sbn_survey_image_service/config/logging.py
index acf0d4a..6b0d619 100644
--- a/sbn_survey_image_service/logging.py
+++ b/sbn_survey_image_service/config/logging.py
@@ -22,7 +22,7 @@ def setup() -> logging.Logger:
"""
- logger: logging.Logger = logging.getLogger('SBN Survey Image Service')
+ logger: logging.Logger = logging.getLogger("SBN Survey Image Service")
logger.handlers = []
@@ -32,7 +32,8 @@ def setup() -> logging.Logger:
logger.handlers = []
formatter: logging.Formatter = logging.Formatter(
- '%(levelname)s %(asctime)s: %(message)s')
+ "%(levelname)s:%(name)s:%(asctime)s: %(message)s"
+ )
console: logging.StreamHandler = logging.StreamHandler()
console.setFormatter(formatter)
@@ -48,8 +49,8 @@ def setup() -> logging.Logger:
handler: logging.Handler
for handler in logger.handlers:
- if hasattr(handler, 'baseFilename'):
- logger.info('Logging to %s', handler.baseFilename)
+ if hasattr(handler, "baseFilename"):
+ logger.info("Logging to %s", handler.baseFilename)
return logger
@@ -66,7 +67,7 @@ def get_logger() -> logging.Logger:
"""
- logger: logging.Logger = logging.getLogger('SBN Survey Image Service')
+ logger: logging.Logger = logging.getLogger("SBN Survey Image Service")
if len(logger.handlers) == 0:
setup()
diff --git a/sbn_survey_image_service/data/__init__.py b/sbn_survey_image_service/data/__init__.py
index 7b4b462..cb28684 100644
--- a/sbn_survey_image_service/data/__init__.py
+++ b/sbn_survey_image_service/data/__init__.py
@@ -1,4 +1,5 @@
# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
"""Data functions."""
from .core import *
diff --git a/sbn_survey_image_service/data/add.py b/sbn_survey_image_service/data/add.py
index 1f73a89..ce13f47 100644
--- a/sbn_survey_image_service/data/add.py
+++ b/sbn_survey_image_service/data/add.py
@@ -9,14 +9,14 @@
import logging
import argparse
from urllib.parse import urlparse, urlunparse
-from typing import Dict, List, Optional, Union
+from typing import Dict, List
import xml.etree.ElementTree as ET
import numpy as np
from sqlalchemy.orm.session import Session
from pds4_tools.reader.read_label import read_label as pds4_read_label
-from ..exceptions import (
+from ..config.exceptions import (
LabelError,
InvalidNEATImage,
InvalidATLASImage,
@@ -27,7 +27,7 @@
from ..services.database_provider import data_provider_session, db_engine
from ..models import Base
from ..models.image import Image
-from ..logging import get_logger
+from ..config.logging import get_logger
def _remove_prefix(s: str, prefix: str):
@@ -52,6 +52,7 @@ def add_label(
session: Session,
base_url: str = "file://",
strip_leading: str = "",
+ relax: bool = False,
) -> bool:
"""Add label and image data to database.
@@ -71,11 +72,13 @@ def add_label(
strip_leading : str, optional
Remove this leading string from the path before forming the URL.
+ relax : bool, optional
+ Set to ``True`` and errors will be logged, but otherwise ignored.
Returns
-------
success : bool
- False, if the target already exists in the database.
+ ``True``, if the label was successfully added.
"""
@@ -86,10 +89,14 @@ def add_label(
im = pds4_image(label_path)
except SBNSISWarning as exc:
logger.warning(exc)
- return False
+ if relax:
+ return False
+ raise exc
except (LabelError, InvalidImageURL) as exc:
logger.error(exc)
- return False
+ if relax:
+ return False
+ raise exc
# make proper URLs
im.label_url = _normalize_url(
@@ -131,7 +138,6 @@ def pds4_image(label_path: str) -> Image:
exc: Exception
try:
- # data: StructureList = pds4_read(label_path, lazy_load=True)
label: ET.ElementTree = pds4_read_label(
label_path, enforce_default_prefixes=True
)
@@ -191,7 +197,7 @@ def pds4_image(label_path: str) -> Image:
fz_compressed = True
if not valid_atlas_image(label):
raise InvalidATLASImage(
- f"{label_path} does not appear to be an " "ATLAS prime image."
+ f"{label_path} does not appear to be an ATLAS prime image."
)
if fz_compressed:
@@ -200,10 +206,10 @@ def pds4_image(label_path: str) -> Image:
return im
-def pds4_pixel_scale(label: ET.ElementTree) -> Union[float, None]:
+def pds4_pixel_scale(label: ET.ElementTree) -> float | None:
"""Compute average pixel scale from Earth_Based_Telescope discipline dictionary."""
- wcs: ET.ElementTree = label.find(
+ wcs: ET.ElementTree | None = label.find(
".//ebt:World_Coordinate_System",
namespaces={"ebt": "http://pds.nasa.gov/pds4/ebt/v1"},
)
@@ -251,7 +257,7 @@ def add_directory(
path: str,
session: Session,
recursive: bool = False,
- extensions: Optional[List[str]] = None,
+ extensions: List[str] | None = None,
**kwargs,
) -> None:
"""Search directory for labels and add to database.
@@ -322,7 +328,7 @@ def _parse_args() -> argparse.Namespace:
action="append",
default=[".xml"],
help=(
- "additional file name extensions to consider" " while searching directories"
+ "additional file name extensions to consider while searching directories"
),
)
parser.add_argument(
@@ -343,7 +349,7 @@ def _parse_args() -> argparse.Namespace:
return parser.parse_args()
-def _main() -> None:
+def __main__() -> None:
args: argparse.Namespace = _parse_args()
logger: logging.Logger = get_logger()
@@ -363,7 +369,3 @@ def _main() -> None:
)
else:
add_label(ld, session, **kwargs)
-
-
-if __name__ == "__main__":
- _main()
diff --git a/sbn_survey_image_service/data/core.py b/sbn_survey_image_service/data/core.py
index 4a8b3e1..a4a974c 100644
--- a/sbn_survey_image_service/data/core.py
+++ b/sbn_survey_image_service/data/core.py
@@ -10,23 +10,7 @@
from requests.models import HTTPError
from urllib.parse import ParseResult, urlparse
-from ..env import ENV
-
-
-def _generate_image_path(*args):
- """Make consistent cutout file name based on MD5 sum of the arguments.
-
-
- Parameters
- ----------
- *args : strings
- Order is important.
-
- """
-
- m = hashlib.md5()
- m.update("".join(args).encode())
- return os.path.join(ENV.SBNSIS_CUTOUT_CACHE, m.hexdigest())
+from ..config.env import ENV
def url_to_local_file(url: str) -> str:
diff --git a/sbn_survey_image_service/data/test/generate.py b/sbn_survey_image_service/data/test/generate.py
index fcb61da..18a5458 100644
--- a/sbn_survey_image_service/data/test/generate.py
+++ b/sbn_survey_image_service/data/test/generate.py
@@ -18,7 +18,7 @@
from sqlalchemy.exc import OperationalError
import astropy.units as u
-from astropy.coordinates import SkyCoord, Angle
+from astropy.coordinates import SkyCoord
from astropy.io import fits
from astropy.wcs import WCS
from astropy.time import Time
@@ -28,7 +28,7 @@
from ...services.database_provider import data_provider_session, db_engine
from ...models import Base
from ...models.image import Image
-from ...env import ENV
+from ...config.env import ENV
def spherical_distribution(N: int) -> np.ndarray:
@@ -83,23 +83,21 @@ def create_data(session, path):
logger: logging.Logger = logging.getLogger(__name__)
- os.system(f'mkdir -p {path}')
+ os.system(f"mkdir -p {path}")
# "size" of each test image: ~10 deg
# for 1 deg...
# N = int(4 * np.pi / ((3600 / 206265)**2))
# N = 41253
- logger.info('Creating ~400 images and labels.')
+ logger.info("Creating ~400 images and labels.")
centers: np.ndarray = np.degrees(spherical_distribution(400))
image_size: int = 300
- pixel_size: float = np.degrees(
- np.sqrt(4 * np.pi / len(centers))
- ) / image_size / 10
+ pixel_size: float = np.degrees(np.sqrt(4 * np.pi / len(centers))) / image_size / 10
xy: np.ndarray = np.mgrid[:image_size, :image_size][::-1]
w: WCS = WCS()
- w.wcs.ctype = 'RA---TAN', 'DEC--TAN'
+ w.wcs.ctype = "RA---TAN", "DEC--TAN"
w.wcs.crpix = image_size // 2, image_size // 2
w.wcs.pc = [[-pixel_size, 0], [0, pixel_size]]
@@ -108,9 +106,11 @@ def create_data(session, path):
cadence: u.Quantity = 45 * u.s
start_time = Time.now()
- template = (resources.files("sbn_survey_image_service.data.test")
- .joinpath("template.xml")
- .read_text())
+ template = (
+ resources.files("sbn_survey_image_service.data.test")
+ .joinpath("template.xml")
+ .read_text()
+ )
c: np.ndarray
for c in centers:
@@ -123,45 +123,44 @@ def create_data(session, path):
start_time += cadence
observation_number += 1
- basename: str = f'test-{observation_number:06d}.fits'
- image_path: str = f'{path}/{basename}'
- label_path: str = image_path.replace('.fits', '.xml')
+ basename: str = f"test-{observation_number:06d}.fits"
+ image_path: str = f"{path}/{basename}"
+ label_path: str = image_path.replace(".fits", ".xml")
if os.path.exists(image_path) and os.path.exists(label_path):
continue
hdu: fits.HDUList = fits.HDUList()
- hdu.append(fits.PrimaryHDU(
- data.astype(np.int32),
- header=w.to_header())
- )
+ hdu.append(fits.PrimaryHDU(data.astype(np.int32), header=w.to_header()))
hdu.writeto(image_path, overwrite=True)
outf: io.IOBase
- with open(label_path, 'w') as outf:
- outf.write(template.format(
- basename=os.path.basename(label_path[:-4]),
- filename=os.path.basename(image_path),
- start_time=start_time.isot,
- stop_time=(start_time + exptime).isot,
- field_id=observation_number,
- ra=[
- coordinates.ra[-1, 0],
- coordinates.ra[-1, -1],
- coordinates.ra[0, 0],
- coordinates.ra[0, -1],
- ],
- dec=[
- coordinates.dec[-1, 0],
- coordinates.dec[-1, -1],
- coordinates.dec[0, 0],
- coordinates.dec[0, -1],
- ],
- center_ra=w.wcs.crval[0],
- center_dec=w.wcs.crval[1],
- center_x=w.wcs.crpix[0],
- center_y=w.wcs.crpix[1],
- pixel_size=pixel_size,
- ))
+ with open(label_path, "w") as outf:
+ outf.write(
+ template.format(
+ basename=os.path.basename(label_path[:-4]),
+ filename=os.path.basename(image_path),
+ start_time=start_time.isot,
+ stop_time=(start_time + exptime).isot,
+ field_id=observation_number,
+ ra=[
+ coordinates.ra[-1, 0],
+ coordinates.ra[-1, -1],
+ coordinates.ra[0, 0],
+ coordinates.ra[0, -1],
+ ],
+ dec=[
+ coordinates.dec[-1, 0],
+ coordinates.dec[-1, -1],
+ coordinates.dec[0, 0],
+ coordinates.dec[0, -1],
+ ],
+ center_ra=w.wcs.crval[0],
+ center_dec=w.wcs.crval[1],
+ center_x=w.wcs.crpix[0],
+ center_y=w.wcs.crpix[1],
+ pixel_size=pixel_size,
+ )
+ )
if observation_number % 1000 == 0:
logger.info(observation_number)
@@ -169,8 +168,8 @@ def create_data(session, path):
add_directory(ENV.TEST_DATA_PATH, session)
logger.info(
- 'Created and added %d test images and their labels to the database.',
- observation_number
+ "Created and added %d test images and their labels to the database.",
+ observation_number,
)
@@ -182,11 +181,7 @@ def create_tables() -> None:
def delete_data(session) -> None:
"""Delete test data from database."""
- (
- session.query(Image)
- .filter(Image.collection == 'test-collection')
- .delete()
- )
+ (session.query(Image).filter(Image.collection == "test-collection").delete())
def exists(session) -> bool:
@@ -198,9 +193,7 @@ def exists(session) -> bool:
try:
results: Any = (
- session.query(Image)
- .filter(Image.collection == 'test-collection')
- .all()
+ session.query(Image).filter(Image.collection == "test-collection").all()
)
except OperationalError:
return False
@@ -210,8 +203,12 @@ def exists(session) -> bool:
im: Image
for im in results:
- if not any((os.path.exists(url_to_local_file(im.image_url)),
- os.path.exists(url_to_local_file(im.label_url)))):
+ if not any(
+ (
+ os.path.exists(url_to_local_file(im.image_url)),
+ os.path.exists(url_to_local_file(im.label_url)),
+ )
+ ):
return False
return True
@@ -219,19 +216,28 @@ def exists(session) -> bool:
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
- description='Add/delete test data to/from SBN Survey Image Service database.',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter
+ description="Add/delete test data to/from SBN Survey Image Service database.",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ parser.add_argument(
+ "--path",
+ default=ENV.TEST_DATA_PATH,
+ help="directory to which to save test data files",
+ )
+ parser.add_argument("--add", action="store_true", help="add/create test data set")
+ parser.add_argument(
+ "--exists",
+ action="store_true",
+ help="test for the existence of the test data set",
+ )
+ parser.add_argument(
+ "--delete", action="store_true", help="delete test data files and database rows"
+ )
+ parser.add_argument(
+ "--no-create-tables",
+ action="store_true",
+ help="do not attempt to create missing database tables",
)
- parser.add_argument('--path', default=ENV.TEST_DATA_PATH,
- help='directory to which to save test data files')
- parser.add_argument('--add', action='store_true',
- help='add/create test data set')
- parser.add_argument('--exists', action='store_true',
- help='test for the existence of the test data set')
- parser.add_argument('--delete', action='store_true',
- help='delete test data files and database rows')
- parser.add_argument('--no-create-tables', action='store_true',
- help='do not attempt to create missing database tables')
args: argparse.Namespace = parser.parse_args()
if len(sys.argv) == 1:
@@ -255,14 +261,13 @@ def _main() -> None:
create_data(session, args.path)
elif args.delete:
delete_data(session)
- logger.info(
- 'Database cleaned, but test files must be removed manually.')
+ logger.info("Database cleaned, but test files must be removed manually.")
elif args.exists:
if exists(session):
- print('Test data set appears to be valid.')
+ print("Test data set appears to be valid.")
else:
- print('Test data set is broken or missing.')
+ print("Test data set is broken or missing.")
-if __name__ == '__main__':
+if __name__ == "__main__":
_main()
diff --git a/sbn_survey_image_service/data/test/template.xml b/sbn_survey_image_service/data/test/template.xml
index 5aabb0c..27c9e01 100644
--- a/sbn_survey_image_service/data/test/template.xml
+++ b/sbn_survey_image_service/data/test/template.xml
@@ -167,7 +167,7 @@
1
1
- {-pixel_size}
+ -{pixel_size}
1
diff --git a/sbn_survey_image_service/scripts/sbnsis.py b/sbn_survey_image_service/scripts/sbnsis.py
index 7a12dc5..5deaf17 100644
--- a/sbn_survey_image_service/scripts/sbnsis.py
+++ b/sbn_survey_image_service/scripts/sbnsis.py
@@ -9,7 +9,7 @@
from argparse import ArgumentParser
from typing import Dict, List, Tuple
-from sbn_survey_image_service.env import ENV, env_example
+from sbn_survey_image_service.config.env import ENV, env_example
class ServiceException(Exception):
@@ -48,12 +48,12 @@ def __init__(self):
if hasattr(self.args, "func"):
try:
- print_color('#' * 72 + "\n")
+ print_color("#" * 72 + "\n")
self.args.func() # run requested function
- print_color("\n" + '#' * 72)
+ print_color("\n" + "#" * 72)
except ServiceException as e:
print_color(str(e), color=Colors.red)
- print_color("\n" + '#' * 72)
+ print_color("\n" + "#" * 72)
exit(1)
else:
parser.print_usage()
@@ -65,8 +65,9 @@ def _check_venv(self) -> None:
def _get_gunicorn_processes(self) -> Tuple[int, int]:
"""Return number of running processes for this virtual environment and the parent PID."""
- all_processes: List[str] = subprocess.check_output(
- ["ps", "-ef"]).decode().splitlines()
+ all_processes: List[str] = (
+ subprocess.check_output(["ps", "-ef"]).decode().splitlines()
+ )
venv: str = os.getenv("VIRTUAL_ENV")
processes: List[str] = [
process for process in all_processes if f"{venv}/bin/gunicorn" in process
@@ -101,7 +102,7 @@ def start_dev(self) -> None:
"--exec",
"python3",
"-m",
- "sbn_survey_image_service.api.app",
+ "sbn_survey_image_service.app",
]
try:
@@ -112,7 +113,7 @@ def start_dev(self) -> None:
def start_production(self) -> None:
cmd: List[str] = [
"gunicorn",
- "sbn_survey_image_service.api.app:app",
+ "sbn_survey_image_service.app:app",
"--workers",
str(ENV.LIVE_GUNICORN_INSTANCES),
"--bind",
@@ -157,8 +158,7 @@ def restart(self) -> None:
os.kill(ppid, signal.SIGWINCH)
ellipsis(10)
- print_color(" - Stopping old service parent process",
- end="", flush=True)
+ print_color(" - Stopping old service parent process", end="", flush=True)
os.kill(ppid, signal.SIGQUIT)
ellipsis(1)
@@ -201,24 +201,28 @@ def status(self, quiet=False) -> Tuple[int, int]:
if n == 0:
if not quiet:
print_color(
- "No sbnsis-service running from this project's virtual environment.")
+ "No sbnsis-service running from this project's virtual environment."
+ )
return 0, 0
print_color(
- f"sbnsis-service is running with {n - 1} workers.\nParent PID: {ppid}")
+ f"sbnsis-service is running with {n - 1} workers.\nParent PID: {ppid}"
+ )
return n, ppid
def rotate_logs(self) -> None:
print_color("Rotating logs.")
- subprocess.check_call([
- "/usr/sbin/logrotate",
- "--force",
- "--state",
- "logrotate.state",
- "logrotate.config",
- ],
- cwd="logging")
+ subprocess.check_call(
+ [
+ "/usr/sbin/logrotate",
+ "--force",
+ "--state",
+ "logrotate.state",
+ "logrotate.config",
+ ],
+ cwd="logging",
+ )
def env_file(self) -> None:
if os.path.exists(".env") and not self.args.print:
@@ -233,8 +237,7 @@ def env_file(self) -> None:
print_color("Wrote new .env file.")
def argument_parser(self) -> ArgumentParser:
- parser: ArgumentParser = ArgumentParser(
- description="SBN Survey Image Service")
+ parser: ArgumentParser = ArgumentParser(description="SBN Survey Image Service")
subparsers = parser.add_subparsers(help="sub-command help")
# start #########
@@ -267,23 +270,29 @@ def argument_parser(self) -> ArgumentParser:
# stop ##########
stop_parser: ArgumentParser = subparsers.add_parser(
- "stop", help="stop the service")
+ "stop", help="stop the service"
+ )
stop_parser.set_defaults(func=self.stop)
# rotate-logs ##########
rotate_logs_parser: ArgumentParser = subparsers.add_parser(
- "rotate-logs", help="force rotate logs")
+ "rotate-logs", help="force rotate logs"
+ )
rotate_logs_parser.set_defaults(func=self.rotate_logs)
# env ##########
env_parser: ArgumentParser = subparsers.add_parser(
- "env", help="create a new .env file")
- env_parser.add_argument("--print", action="store_true",
- help="print the defaults, but do not save to .env")
+ "env", help="create a new .env file"
+ )
+ env_parser.add_argument(
+ "--print",
+ action="store_true",
+ help="print the defaults, but do not save to .env",
+ )
env_parser.set_defaults(func=self.env_file)
return parser
-if __name__ == "__main__":
+def __main__():
SBNSISService()
diff --git a/sbn_survey_image_service/services/__init__.py b/sbn_survey_image_service/services/__init__.py
index 3075143..e673217 100644
--- a/sbn_survey_image_service/services/__init__.py
+++ b/sbn_survey_image_service/services/__init__.py
@@ -1,8 +1,2 @@
# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""Database and image services."""
-
-from . import database_provider, label, image, metadata
-from .database_provider import data_provider_session
-from .label import *
-from .image import *
-from .metadata import *
diff --git a/sbn_survey_image_service/services/database_provider.py b/sbn_survey_image_service/services/database_provider.py
index fd32e9b..25364fa 100755
--- a/sbn_survey_image_service/services/database_provider.py
+++ b/sbn_survey_image_service/services/database_provider.py
@@ -13,14 +13,16 @@
from sqlalchemy.exc import SQLAlchemyError, DBAPIError
from sqlalchemy.pool import NullPool
-from ..env import ENV
+from ..config.env import ENV
# Build URI and instantiate data-provider service
db_engine_URI: str = (
f"{ENV.DB_DIALECT}://{ENV.DB_USERNAME}:{ENV.DB_PASSWORD}@{ENV.DB_HOST}"
- f"/{ENV.DB_DATABASE}")
+ f"/{ENV.DB_DATABASE}"
+)
db_engine: Engine = sqlalchemy.create_engine(
- db_engine_URI, poolclass=NullPool, pool_recycle=3600, pool_pre_ping=True)
+ db_engine_URI, poolclass=NullPool, pool_recycle=3600, pool_pre_ping=True
+)
db_session: scoped_session = scoped_session(sessionmaker(bind=db_engine))
diff --git a/sbn_survey_image_service/services/image.py b/sbn_survey_image_service/services/image.py
index dae1c0d..0a53754 100644
--- a/sbn_survey_image_service/services/image.py
+++ b/sbn_survey_image_service/services/image.py
@@ -13,8 +13,8 @@
from .database_provider import data_provider_session, Session
from ..data import url_to_local_file, generate_cache_filename
from ..models.image import Image
-from ..exceptions import InvalidImageID, ParameterValueError, FitscutError
-from ..env import ENV
+from ..config.exceptions import InvalidImageID, ParameterValueError, FitscutError
+from ..config.env import ENV
FORMATS = {"png": "--png", "jpeg": "--jpg"}
diff --git a/sbn_survey_image_service/services/label.py b/sbn_survey_image_service/services/label.py
index 1056332..6024e6f 100644
--- a/sbn_survey_image_service/services/label.py
+++ b/sbn_survey_image_service/services/label.py
@@ -1,9 +1,7 @@
# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""Data product label service."""
-__all__ = [
- 'label_query'
-]
+__all__ = ["label_query"]
import os
from typing import Tuple
@@ -12,7 +10,7 @@
from .database_provider import data_provider_session, Session
from ..models.image import Image
-from ..exceptions import InvalidImageID
+from ..config.exceptions import InvalidImageID
from ..data import url_to_local_file
@@ -23,11 +21,9 @@ def label_query(obs_id: str) -> Tuple[str, str]:
exc: Exception
try:
label_url: str = (
- session.query(Image.label_url)
- .filter(Image.obs_id == obs_id)
- .one()[0]
+ session.query(Image.label_url).filter(Image.obs_id == obs_id).one()[0]
)
except NoResultFound as exc:
- raise InvalidImageID('Image ID not found in database.') from exc
+ raise InvalidImageID("Image ID not found in database.") from exc
return url_to_local_file(label_url), os.path.basename(label_url)
diff --git a/sbn_survey_image_service/services/metadata.py b/sbn_survey_image_service/services/metadata.py
index 9d17fef..6018d75 100644
--- a/sbn_survey_image_service/services/metadata.py
+++ b/sbn_survey_image_service/services/metadata.py
@@ -1,24 +1,22 @@
# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""Data product metdata service."""
-__all__ = [
- 'metadata_query',
- 'metadata_summary'
-]
+__all__ = ["metadata_query", "metadata_summary"]
-from typing import Any, List, Optional, Tuple
+from typing import Any, List, Tuple
from .database_provider import data_provider_session, Session
from ..models.image import Image
-def metadata_query(collection: Optional[str] = None,
- facility: Optional[str] = None,
- instrument: Optional[str] = None,
- dptype: Optional[str] = None,
- format: str = 'fits',
- maxrec: int = 100,
- offset: int = 0,
- ) -> Tuple[int, List[dict]]:
+def metadata_query(
+ collection: str | None = None,
+ facility: str | None = None,
+ instrument: str | None = None,
+ dptype: str | None = None,
+ format: str = "fits",
+ maxrec: int = 100,
+ offset: int = 0,
+) -> Tuple[int, List[dict]]:
"""Query database for image metadata.
@@ -27,14 +25,14 @@ def metadata_query(collection: Optional[str] = None,
count : int
Total number of matches.
- matches : list
+ matches : list of dict
+ The matches.
"""
matches: List[dict] = []
session: Session
- # exc: Exception
with data_provider_session() as session:
query: Any = session.query(Image)
if collection is not None:
@@ -53,45 +51,59 @@ def metadata_query(collection: Optional[str] = None,
query = query.offset(offset)
- # try:
- # images: List[Image] = query.all()
- # except NoResultFound as exc:
- # return []
-
images: List[Image] = query.all()
for im in images:
- matches.append({
- 'obs_id': im.obs_id,
- 'collection': im.collection,
- 'facility': im.facility,
- 'instrument': im.instrument,
- 'dptype': im.data_product_type,
- 'calibration_level': im.calibration_level,
- 'target': im.target,
- 'pixel_scale': im.pixel_scale,
- 'access_url': f'https://sbnsurveys.astro.umd.edu/images/{im.obs_id}?format={format}'
- })
+ matches.append(
+ {
+ "obs_id": im.obs_id,
+ "collection": im.collection,
+ "facility": im.facility,
+ "instrument": im.instrument,
+ "dptype": im.data_product_type,
+ "calibration_level": im.calibration_level,
+ "target": im.target,
+ "pixel_scale": im.pixel_scale,
+ "access_url": f"https://sbnsurveys.astro.umd.edu/images/{im.obs_id}?format={format}",
+ }
+ )
return count, matches
def metadata_summary() -> List[dict]:
+ """Summarize the database holdings.
+
+
+ Returns
+ -------
+ summary : list of dict
+
+ """
+
session: Session
summary: List[dict] = []
with data_provider_session() as session:
- rows: List[dict] = session.query(Image.collection, Image.facility,
- Image.instrument).distinct().all()
+ rows: List[dict] = (
+ session.query(Image.collection, Image.facility, Image.instrument)
+ .distinct()
+ .all()
+ )
for collection, facility, instrument in rows:
- count: int = (session.query(Image)
- .filter(Image.collection == collection)
- .filter(Image.facility == facility)
- .filter(Image.instrument == instrument)
- ).count()
- summary.append({'collection': collection,
- 'facility': facility,
- 'instrument': instrument,
- 'count': count})
+ count: int = (
+ session.query(Image)
+ .filter(Image.collection == collection)
+ .filter(Image.facility == facility)
+ .filter(Image.instrument == instrument)
+ ).count()
+ summary.append(
+ {
+ "collection": collection,
+ "facility": facility,
+ "instrument": instrument,
+ "count": count,
+ }
+ )
return summary
diff --git a/sbn_survey_image_service/test/test_services.py b/sbn_survey_image_service/test/test_services.py
index 9ddb442..832419b 100644
--- a/sbn_survey_image_service/test/test_services.py
+++ b/sbn_survey_image_service/test/test_services.py
@@ -2,7 +2,6 @@
"""Test services using test data set."""
import os
-from hashlib import md5
import pytest
from sqlalchemy.orm.session import Session
import numpy as np
@@ -10,9 +9,11 @@
from ..data.test import generate
from ..data import generate_cache_filename
-from ..services import data_provider_session, image_query, label_query
-from ..env import ENV
-from ..exceptions import InvalidImageID, ParameterValueError
+from ..services.database_provider import data_provider_session
+from ..services.image import image_query
+from ..services.label import label_query
+from ..config.env import ENV
+from ..config.exceptions import InvalidImageID, ParameterValueError
@pytest.fixture(autouse=True)
@@ -111,7 +112,7 @@ def test_image_query_cutout():
expected_path: str = generate_cache_filename(
"file://" + os.path.join(ENV.TEST_DATA_PATH, "test-000102.fits"),
- "urn:nasa:pds:survey:test-collection:test-000102",
+ "urn:nasa:pds:survey:test-collection:test-000102",
str(ra),
str(dec),
str(size),