Merge pull request #13 from Small-Bodies-Node/drop-fitscut

Drop fitscut

_precommit_hook

@@ -14,6 +14,5 @@ if [ $DONT_FORMAT_ON_CODE_COMMIT ]; then
     """
 else
     # Auto-format all python scripts
-    .venv/bin/autopep8 -ir sbn_survey_image_service/**
-    .venv/bin/autopep8 -ir tests/**
+    .venv/bin/black sbn_survey_image_service/**
 fi

_setup_development.sh → _setup_development

@@ -59,14 +59,6 @@ ${reset_color}"""
 pip install --upgrade -q -q -q pip setuptools wheel
 pip install -e .[recommended,dev,test,docs]
 
-if [[ ! -e $VIRTUAL_ENV/bin/fitscut ]]; then
-    echo -e """${cyan}
-    Installing fitscut
-${reset_color}
-"""
-    ./_install_fitscut
-fi
-
 ### Link git pre-commit-hook script
 ln -fs $PWD/_precommit_hook $PWD/.git/hooks/pre-commit
 

_setup_production.sh

@@ -62,14 +62,6 @@ latest=$(git describe --tags "$(git rev-list --tags --max-count=1)")
 git checkout $latest
 pip install .[recommended]
 
-if [[ ! -e $VIRTUAL_ENV/bin/fitscut ]]; then
-    echo -e """${cyan}
-    Installing fitscut
-${reset_color}
-"""
-    ./_install_fitscut
-fi
-
 ### Link git pre-commit-hook script
 ln -fs $PWD/_precommit_hook $PWD/.git/hooks/pre-commit
 

_tests → _test

@@ -1,7 +1,7 @@
 #! /bin/bash
 
 # Run test scripts
-pytest --verbose tests sbn_survey_image_service \
+pytest --verbose sbn_survey_image_service \
   --cov=sbn_survey_image_service \
   --cov-report=html \
   --remote-data \

docs/adding-data.rst

@@ -74,12 +74,10 @@ in ``sbn_survey_image_service/data/add.py``, e.g., ``pds4_image()``. For
 example, the NEAT survey PDS4 data set v1.0 does not have pixel scale in the
 label, so we have hard coded it into the ``pds4_image`` function.
 
-
-Caveats
--------
-
 It is assumed that survey images are FITS-compatible with a World Coordinate
-System defined for a standard sky reference frame (ICRS). The cutout service
-uses the FITS header, not the PDS labels, to define the sub-frame. This is a
-limitation from using fitscut.  **fitscut is planned to be replaced in a future
-version** to better support metadata sourced from the PDS labels.
+System defined for a standard sky reference frame (ICRS).
+
+.. attention::
+
+   The cutout service uses the FITS header, not the PDS labels, to define the sub-frame. Sourcing the
+   WCS from the labels will be addressed in a future version.

docs/development.rst

@@ -1,18 +1,60 @@
 Development
 ===========
 
-The bash script ``_setup_development`` will create and setup a virtual
-environment for development.  Use that now, or follow :doc:`install`, replacing
-the package installation step as directed below.
+The bash script ``_setup_development`` will create and setup a virtual environment for development.  Use that now, or follow :doc:`install`, replacing the package installation step as directed below.
 
-For development, install the package with the "dev", "test", and "docs" options.
-Also, it is beneficial to install the package in "editable" mode with the "-e"
-option to pip:
+For development, install the package with the "dev", "test", and "docs" options. Also, it is beneficial to install the package in "editable" mode with the "-e" option to pip:
 
 .. code:: bash
 
     pip install -e .[recommended,dev,test,docs]
 
-If you have ``nodemon`` globally installed, then you can develop your api code
-and have it automatically update on changes by running ``sbnsis start --dev``.
-Otherwise, just run ``python -m sbn_survey_image_service.app``.
+
+Development mode
+----------------
+
+If you have ``nodemon`` globally installed, then you can develop your api code and have it automatically update on changes by running ``sbnsis start --dev``.  Otherwise, just run ``python -m sbn_survey_image_service.app``.
+
+
+Testing
+-------
+
+Testing requires a test data set (images and PDS4 labels).  The test data set may be generated with the provided script:
+
+.. code:: 
+
+    $ python3 -m sbn_survey_image_service.data.test.generate
+    usage: generate.py [-h] [--path PATH] [--add] [--exists] [--delete]
+                    [--no-create-tables]
+
+    Add/delete test data to/from SBN Survey Image Service database.
+
+    options:
+    -h, --help          show this help message and exit
+    --path PATH         directory to which to save test data files (default:
+                        /sbnsurveys/src/sbn-survey-image-
+                        service/data/test)
+    --add               add/create test data set (default: False)
+    --exists            test for the existence of the test data set (default:
+                        False)
+    --delete            delete test data files and database rows (default:
+                        False)
+    --no-create-tables  do not attempt to create missing database tables
+                        (default: False)
+
+Run the `--add` option to generate the files and populate the database:
+
+.. warning::
+
+    `--add` will use the database configuration in your `.env` file.  It isn't harmful to add the test data set to a production database, but this may not be desired.  Testing data may be removed from the database with the `--delete` option.
+
+.. code::
+
+    $ python3 -m sbn_survey_image_service.data.test.generate --add
+    INFO:__main__:Creating ~400 images and labels.
+    INFO:SBN Survey Image Service:2024-11-08 09:06:47,245: Logging to /sbnsurveys/src/sbn-survey-image-service/logging/sbnsis.log
+    INFO:SBN Survey Image Service:2024-11-08 09:06:47,245: Searching directory /sbnsurveys/src/sbn-survey-image-service/data/test
+    INFO:SBN Survey Image Service:2024-11-08 09:06:47,912: Searched 1 directories, found 404 labels, 404 processed.
+    INFO:__main__:Created and added 404 test images and their labels to the database.
+
+The script `_test` will run the tests with `pytest`.  Coverage reports will be saved in HTML format to `htmlcov/`.  Tests requiring network access are run by default (using the --remote-data option to pytest).

docs/install.rst

@@ -1,20 +1,27 @@
 Installation and Setup
 ======================
 
-Select and install your choice of database backend, either sqlite3 or
-postgresql.  If using postgresql, it is recommended to have two separate users
-with access: one that can perform database maintenance (e.g., INSERT and UPDATE
-permissions), and another that will be limited to read-only access.
+Select and install your choice of database backend, either sqlite3 or postgresql.  If using postgresql, it is recommended to have two separate users with access: one that can perform database maintenance (e.g., INSERT and UPDATE permissions), and another that will be limited to read-only access.
 
 
 Requirements
 ------------
 
-All python requirements are managed by the pyproject.toml file and automatically
-installed with pip below.  In addition,
-[libtool](https://www.gnu.org/software/libtool/) and fitscut are needed.  Follow
-your typical system installation instructions for libtool.  The installation of
-fitscut is described below.
+All Python requirements are managed by the pyproject.toml file and automatically
+installed with pip below.  Major Python dependencies are:
+
+* astropy
+* connexion
+* gunicorn
+* pds4_tools
+* Pillow
+* requests
+* SQLAlchemy
+
+Other dependencies:
+
+* logrotate
+* nodemon - for running in development mode
 
 
 SBN SIS
@@ -48,18 +55,6 @@ dependencies:
    pip install .[recommended]
 
 
-fitscut
--------
-
-The SIS uses the fitscut utility for generating cutouts and web (e.g., JPEG)
-images.  Build and install it to the virtual environment.  There is a bash
-script that can do this automatically for you:
-
-.. code:: bash
-
-   bash _install_fitscut
-
-
 SIS configuration
 -----------------
 

docs/service.rst

@@ -33,55 +33,40 @@ Start the service in production mode:
 
    sbnsis start
 
-The app will launch as a background process with the gunicorn WSGI server. The
-number of workers is controlled with the env variable
-``LIVE_GUNICORN_INSTANCES``. If you have trouble getting gunicorn to work,
-running in non-daemon mode may help with debugging:
+The app will launch as a background process with the gunicorn WSGI server. The number of workers is controlled with the env variable ``LIVE_GUNICORN_INSTANCES``. If you have trouble getting gunicorn to work, running in non-daemon mode may help with debugging:
 
 .. code:: bash
 
     sbnsis start --no-daemon
 
-See :doc:`adding-data` for instructions on how to add data to the database.  The
-service does not need to be running to add data.
+See :doc:`adding-data` for instructions on how to add data to the database.  The service does not need to be running to add data.
 
-It is recommended that you make the gunicorn-powered server accessible to the
-outside world by proxy-passing requests through an HTTPS-enabled web server like
-Apache.
+It is recommended that you make the gunicorn-powered server accessible to the outside world by proxy-passing requests through an HTTPS-enabled web server like Apache.
 
 
 REST API
 --------
 
-Whether running in development or deployment modes, the Swagger documentation
-for the REST API is available at ``http://localhost:API_PORT/BASE_HREF/ui``,
-where ``API_PORT`` and ``BASE_HREF`` is defined in your ``.env``.
+Whether running in development or deployment modes, the Swagger documentation for the REST API is available at ``http://localhost:API_PORT/BASE_HREF/ui``, where ``API_PORT`` and ``BASE_HREF`` is defined in your ``.env``.
 
 
 Logging
 -------
 
-Application error and informational logging is sent to the standard error stream
-(stderr) and the file specified by the ``SBNSIS_LOG_FILE`` environment variable.
+Application error and informational logging is sent to the standard error stream (stderr) and the file specified by the ``SBNSIS_LOG_FILE`` environment variable.  If this is set to use the `logging/` directory in the repository, then the `sbnsis` tool will be able to automatically rotate logs using `logrotate` and the `logging/logrotate.config` file.  If `SBNSIS_LOG_FILE` is set to another location, the `logrotate.config` file should be edited, or else log rotation will not work.  Successful requests will produce two log items: the parameters and the results as JSON-formatted strings. The items are linked by a randomly generated job ID:
 
-Successful requests will produce two log items: the parameters and the results
-as JSON-formatted strings. The items are linked by a randomly generated job ID:
+.. code-bock:: text
+   INFO 2021-02-17 14:10:16,960: {"job_id": "013f7515aa074ee58ad5929c8391a366", "id": "urn:nasa:pds:gbo.ast.neat.survey:data_tricam:p20021023_obsdata_20021023113833a", "ra": 47.4495603, "dec": 32.9424075, "size": "5arcmin", "format": "fits", "download": true}
+   INFO 2021-02-17 14:10:18,339: {"job_id": "013f7515aa074ee58ad5929c8391a366", "filename": "/hylonome3/transient/tmpw8s8qj1b.fits", "download_filename": "20021023113833a.fit_47.4495632.94241_5arcmin.fits", "mime_type": "image/fits"}
 
-```
-INFO 2021-02-17 14:10:16,960: {"job_id": "013f7515aa074ee58ad5929c8391a366", "id": "urn:nasa:pds:gbo.ast.neat.survey:data_tricam:p20021023_obsdata_20021023113833a", "ra": 47.4495603, "dec": 32.9424075, "size": "5arcmin", "format": "fits", "download": true}
-INFO 2021-02-17 14:10:18,339: {"job_id": "013f7515aa074ee58ad5929c8391a366", "filename": "/hylonome3/transient/tmpw8s8qj1b.fits", "download_filename": "20021023113833a.fit_47.4495632.94241_5arcmin.fits", "mime_type": "image/fits"}
-```
 
-OpenAPI errors (e.g., invalid parameter values from the user) are not logged.
-Internal code errors will be logged with a code traceback.
+OpenAPI errors (e.g., invalid parameter values from the user) are not logged.  Internal code errors will be logged with a code traceback.
 
 
 Updating SBNSIS
 ---------------
 
-For minor updates that only require a restart of the server, first tag a release
-on Github with the new version, e.g., v0.3.3.  Then, pull the updated code and
-the new tag, upgrade the SIS, and restart the service:
+For minor updates that only require a restart of the server, first tag a release on Github with the new version, e.g., v0.3.3.  Then, pull the updated code and the new tag, upgrade the SIS, and restart the service:
 
 .. code:: bash
 

pyproject.toml

@@ -5,10 +5,14 @@ readme = "readme.md"
 authors = [{ name = "Michael S. P. Kelley", email = "[email protected]" }]
 license = { text = "BSD 3-Clause License" }
 dynamic = ["version"]
-requires-python = ">= 3.10"
+requires-python = ">=3.11"
 
 dependencies = [
     "astropy>=6.0",
+    "fsspec>=2024.10.0",
+    "aiohttp>=3.10",
+    "requests>=2.32",
+    "Pillow>=11.0",
     "connexion[flask,swagger-ui,uvicorn]~=3.0",
     "gunicorn~=21.2",
     "pds4_tools==1.3",
@@ -19,7 +23,7 @@ dependencies = [
 
 [project.optional-dependencies]
 recommended = ["psycopg2-binary>=2.8"]
-dev = ["autopep8", "mypy", "pycodestyle"]
+dev = ["black", "mypy", "pycodestyle"]
 test = ["pytest>=7.0", "pytest-cov>=3.0"]
 docs = ["sphinx", "sphinx-automodapi", "numpydoc"]
 

readme.md

@@ -10,13 +10,11 @@ The [SBN Survey Image Service](https://sbnsurveys.astro.umd.edu/api/ui) (SIS) is
 
 ## Code Features
 
-- Uses [fitscut](https://github.com/spacetelescope/fitscut) for image cutouts and JPEG/PNG generation
 - Flask API layer
 - OpenAPI spec with Connexion and Swagger
 - Gunicorn for production deployment
 - Backed by Postgresql or Sqlite3
 
 ## Documentation
 
-Documentation is provided in the ``/docs`` directory, and at
-[readthedocs](https://sbn-survey-image-service.readthedocs.io).
+Documentation is provided in the ``/docs`` directory, and at [readthedocs](https://sbn-survey-image-service.readthedocs.io).

sbn_survey_image_service/__init__.py

@@ -7,8 +7,6 @@
 # make cache directory set umask
 from .config import exceptions
 from .config import env
-from . import services
-from . import models
 
 try:
     __version__ = _version(__name__)

sbn_survey_image_service/app.py

@@ -69,5 +69,4 @@ def handle_other_error(error: Exception):
     # for development
     logger.info("Running " + ENV.APP_NAME)
     logger.info(application.url_map)
-    app.run("sbn_survey_image_service.app:app",
-            host=ENV.API_HOST, port=ENV.API_PORT)
+    app.run("sbn_survey_image_service.app:app", host=ENV.API_HOST, port=ENV.API_PORT)

sbn_survey_image_service/config/exceptions.py

@@ -39,12 +39,6 @@ class ParameterValueError(SBNSISException):
     code = 400
 
 
-class FitscutError(SBNSISException):
-    """Error processing data with fitscut."""
-
-    code = 500
-
-
 class DatabaseError(SBNSISException):
     """Database error."""