From e7f91643b23c6a0ac9f7b58a598376fdf69f77f1 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Tue, 14 Sep 2021 15:19:58 -0500 Subject: [PATCH] WIP NASA-PDS/devops#10 --- .dockerignore | 1 + .gitattributes | 1 + .github/CODEOWNERS | 47 +- .github/workflows/stable-cicd.yaml | 34 +- .github/workflows/unstable-cicd.yaml | 14 +- .gitignore | 78 +- .pre-commit-config.yaml | 44 + LICENSE.txt | 2 +- NOTICE.txt | 2 +- README.md | 40 +- SCOPE.md | 2 +- docs/design/pds-doi-service-srd.md | 10 +- docs/operations/OSTI.md | 24 +- docs/requirements/0.0.6-dev/REQUIREMENTS.md | 26 +- docs/requirements/0.0.7-dev/REQUIREMENTS.md | 26 +- docs/requirements/0.0.8-dev/REQUIREMENTS.md | 30 +- docs/requirements/1.0.0/REQUIREMENTS.md | 30 +- docs/requirements/v1.2.0-dev/REQUIREMENTS.md | 80 +- docs/requirements/v21.2.0/REQUIREMENTS.md | 80 +- docs/requirements/v9.8.7/REQUIREMENTS.md | 58 +- docs/source/_static/theme_overrides.css | 13 + docs/source/conf.py | 7 +- docs/source/index.rst | 14 +- docs/source/installation/index.rst | 6 +- docs/source/support/index.rst | 1 - docs/source/usage/index.rst | 1 - features/reserve_doi.feature | 14 +- features/steps/steps.py | 5 - input/DOI_Release_20210216_from_draft.json | 1 - input/DOI_Reserved_GEO_200318.csv | 2 +- input/DOI_Reserved_PDS3.csv | 1 - pyproject.toml | 9 + setup.cfg | 151 ++- setup.py | 93 +- src/pds_doi_service/__init__.py | 9 +- src/pds_doi_service/_version.py | 135 +-- src/pds_doi_service/api/__init__.py | 1 - src/pds_doi_service/api/__main__.py | 49 +- .../api/controllers/dois_controller.py | 260 +++-- src/pds_doi_service/api/encoder.py | 3 +- src/pds_doi_service/api/models/__init__.py | 5 +- src/pds_doi_service/api/models/base_model_.py | 21 +- src/pds_doi_service/api/models/doi_record.py | 64 +- src/pds_doi_service/api/models/doi_summary.py | 41 +- .../api/models/label_payload.py | 53 +- .../api/models/labels_payload.py | 20 +- src/pds_doi_service/api/test/__init__.py | 8 +- src/pds_doi_service/api/test/_base.py | 7 +- .../api/test/data/datacite/reserve_record | 2 +- .../api/test/data/osti/draft_record | 1 - .../api/test/data/osti/output.xml | 1 - .../api/test/data/osti/release_record | 1 - .../api/test/data/osti/reserve_record | 1 - .../api/test/test_dois_controller.py | 936 ++++++++---------- src/pds_doi_service/api/util.py | 25 +- src/pds_doi_service/core/actions/__init__.py | 5 +- src/pds_doi_service/core/actions/action.py | 25 +- src/pds_doi_service/core/actions/check.py | 211 ++-- src/pds_doi_service/core/actions/draft.py | 166 ++-- src/pds_doi_service/core/actions/list.py | 120 ++- src/pds_doi_service/core/actions/release.py | 144 +-- src/pds_doi_service/core/actions/reserve.py | 136 +-- .../core/actions/test/__init__.py | 14 +- .../core/actions/test/check_test.py | 178 ++-- .../core/actions/test/draft_test.py | 207 ++-- .../core/actions/test/list_test.py | 75 +- .../core/actions/test/release_test.py | 146 ++- .../core/actions/test/reserve_test.py | 159 ++- src/pds_doi_service/core/cmd/pds_doi_cmd.py | 12 +- src/pds_doi_service/core/db/doi_database.py | 239 +++-- src/pds_doi_service/core/db/test/__init__.py | 8 +- .../core/db/test/doi_database_test.py | 232 +++-- src/pds_doi_service/core/entities/doi.py | 48 +- src/pds_doi_service/core/input/exceptions.py | 32 +- src/pds_doi_service/core/input/input_util.py | 153 ++- src/pds_doi_service/core/input/node_util.py | 38 +- src/pds_doi_service/core/input/pds4_util.py | 231 ++--- .../core/input/test/__init__.py | 11 +- .../core/input/test/input_util_test.py | 112 +-- .../core/input/test/read_bundle.py | 12 +- .../core/input/test/read_remote_bundle.py | 20 +- .../core/input/test/read_xls.py | 7 +- .../core/outputs/datacite/__init__.py | 2 - .../core/outputs/datacite/datacite_record.py | 56 +- .../outputs/datacite/datacite_validator.py | 55 +- .../outputs/datacite/datacite_web_client.py | 69 +- .../outputs/datacite/datacite_web_parser.py | 190 ++-- .../core/outputs/doi_record.py | 12 +- .../core/outputs/doi_validator.py | 154 +-- .../core/outputs/osti/IAD3_schematron.sch | 15 +- .../core/outputs/osti/__init__.py | 7 +- .../core/outputs/osti/osti_record.py | 78 +- .../core/outputs/osti/osti_validator.py | 49 +- .../core/outputs/osti/osti_web_client.py | 88 +- .../core/outputs/osti/osti_web_parser.py | 284 +++--- src/pds_doi_service/core/outputs/service.py | 63 +- .../core/outputs/service_validator.py | 5 +- .../core/outputs/test/__init__.py | 8 +- .../core/outputs/test/datacite_test.py | 179 ++-- .../core/outputs/test/doi_validator_test.py | 331 ++++--- .../core/outputs/test/osti_test.py | 157 +-- .../core/outputs/transaction.py | 31 +- .../core/outputs/transaction_builder.py | 45 +- .../core/outputs/transaction_on_disk.py | 25 +- .../core/outputs/web_client.py | 62 +- .../core/outputs/web_parser.py | 11 +- .../core/references/contributors.py | 20 +- .../core/references/test/__init__.py | 8 +- .../core/references/test/contributors_test.py | 19 +- src/pds_doi_service/core/util/cmd_parser.py | 49 +- .../core/util/config_parser.py | 25 +- .../core/util/doi_xml_differ.py | 326 +++--- src/pds_doi_service/core/util/emailer.py | 13 +- src/pds_doi_service/core/util/general_util.py | 12 +- .../util/initialize_production_deployment.py | 274 ++--- .../core/util/keyword_tokenizer.py | 28 +- .../core/util/test/__init__.py | 8 +- .../core/util/test/config_parser_test.py | 59 +- src/pds_doi_service/test.py | 16 +- tests/data/valid_browsecoll_doi.xml | 1 - tests/data/valid_bundle_doi.xml | 1 - tests/data/valid_calibcoll_doi.xml | 1 - tests/data/valid_datacoll_doi.xml | 1 - tests/data/valid_docucoll_doi.xml | 1 - tests/end_to_end/reserve.csv | 2 +- tests/reserve_ok/output.xml | 2 +- tox.ini | 23 + 127 files changed, 4109 insertions(+), 4087 deletions(-) create mode 100644 .gitattributes create mode 100644 .pre-commit-config.yaml create mode 100644 docs/source/_static/theme_overrides.css create mode 100644 pyproject.toml create mode 100644 tox.ini diff --git a/.dockerignore b/.dockerignore index d3e4222e..8389db4e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,5 +4,6 @@ doi_temp.db transaction_history/ **/test* **/.DS_Store +**/._* **/__pycache__ **/.pytest_cache diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..114a2e6c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +src/pds/my_pds_module/_version.py export-subst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a524712..1471c5bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,19 +1,46 @@ -# This is a comment. +# 📀 Code Owners +# +# Copyright © 2021, California Institute of Technology ("Caltech"). +# U.S. Government sponsorship acknowledged. +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# • Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# • Redistributions must reproduce the above copyright notice, this list of +# conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# • Neither the name of Caltech nor its operating division, the Jet Propulsion +# Laboratory, nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# # Each line is a file pattern followed by one or more owners. - +# # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence, # @global-owner1 and @global-owner2 will be requested for # review when someone opens a pull request. - -# *************************************************************** # -# Go to https://github.com/orgs/pds-data-dictionaries/teams to -# find out more information about your applicable team - -* @NASA-PDS/pdsen-python-committers +# Go to https://github.com/orgs/NASA-PDS/teams to find out about our teams -# ************************************************************** +* @NASA-PDS/pdsen-python-committers -# For more information on populating this file, go to +# For more information on populating this file, check out # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 91e5f9b9..2954b585 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -53,17 +53,28 @@ jobs: token: ${{secrets.ADMIN_GITHUB_TOKEN}} fetch-depth: 0 - - name: Update default configuration + name: 💵 Python Cache + uses: actions/cache@v2 + with: + path: ~/.cache/pip + # The "key" used to indicate a set of cached files is the operating system runner + # plus "py" for Python-specific builds, plus a hash of the wheels, plus "pds" because + # we pds-prefix everything with "pds" in PDS! 😅 + key: pds-${{runner.os}}-py-${{hashFiles('**/*.whl')}} + # To restore a set of files, we only need to match a prefix of the saved key. + restore-keys: pds-${{runner.os}}-py- + - + name: 🔧 Update default configuration run: | - import sys - import os - print("Running python version {}".format(sys.version)) - conf_file = "pds_doi_service.ini" - print("Create config file for unit test {}".format(conf_file)) - with open(conf_file, "w") as f: - f.write("[OSTI]\n") - f.write("user = {}\n".format("${{secrets.osti_login}}")) - f.write("password = {}\n".format("${{secrets.osti_password}}")) + import sys + import os + print("Running python version {}".format(sys.version)) + conf_file = "pds_doi_service.ini" + print("Create config file for unit test {}".format(conf_file)) + with open(conf_file, "w") as f: + f.write("[OSTI]\n") + f.write("user = {}\n".format("${{secrets.osti_login}}")) + f.write("password = {}\n".format("${{secrets.osti_password}}")) shell: python - name: 🤠 Roundup @@ -74,3 +85,6 @@ jobs: pypi_username: ${{secrets.PYPI_USERNAME}} pypi_password: ${{secrets.PYPI_PASSWORD}} ADMIN_GITHUB_TOKEN: ${{secrets.ADMIN_GITHUB_TOKEN}} + +... +# -*- mode: yaml; indent: 4; fill-column: 120; coding: utf-8 -*- diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 6e87dabc..ea238513 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -45,6 +45,7 @@ on: jobs: unstable-assembly: name: 🧩 Unstable Assembly + if: github.actor != 'pdsen-ci' runs-on: ubuntu-latest steps: - @@ -54,6 +55,17 @@ jobs: lfs: true fetch-depth: 0 token: ${{secrets.ADMIN_GITHUB_TOKEN}} + - + name: 💵 Python Cache + uses: actions/cache@v2 + with: + path: ~/.cache/pip + # The "key" used to indicate a set of cached files is the operating system runner + # plus "py" for Python-specific builds, plus a hash of the wheels, plus "pds" because + # we pds-prefix everything with "pds" in PDS! 😅 + key: pds-${{runner.os}}-py-${{hashFiles('**/*.whl')}} + # To restore a set of files, we only need to match a prefix of the saved key. + restore-keys: pds-${{runner.os}}-py- - name: Update default configuration run: | @@ -77,5 +89,5 @@ jobs: pypi_password: ${{secrets.TEST_PYPI_PASSWORD}} ADMIN_GITHUB_TOKEN: ${{secrets.ADMIN_GITHUB_TOKEN}} - +... # -*- mode: yaml; indent: 4; fill-column: 120; coding: utf-8 -*- diff --git a/.gitignore b/.gitignore index 7fcb6d64..c2617664 100644 --- a/.gitignore +++ b/.gitignore @@ -1,34 +1,86 @@ +# Python build, virtual environments, and buildouts +venv +__pycache__/ +dist/ +build/ +*.egg-info +.*.cfg +develop-eggs/ +.python-eggs/ +.eggs/ +pip-selfcheck.json +.python-version + +# Python testing artifacts +.coverage +htmlcov +.tox/ + # Object files -*.pyc +*.o +*.pkl +*.py[ocd] -# IntelliJ files +# Libraries +*.lib +*.a + +# Eclipse files +.settings/ +*.project +*.classpath + +# Editor support .idea/ *.iml +.vscode +*.sublime-* + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib +lib/ +lib64/ + +# Executables +*.exe +*.out +*.app # Temporary files *~ +.*.swp +var/ # other stuff -dist/ -pds_doi_service.ini +*.log +*.xpr +bin/ +.*.swp + +# OS-specific artifacts .DS_Store +._* + +# Exclusions +!.coveragerc +!.editorconfig +!.gitattributes +!.gitignore +!.gitkeep + +# Project-specific +pds_doi_service.ini allure/ -pds_doi_service/api/pds_doi_api.egg-info/ -pds_doi_core.egg-info/ -pds_doi_service.egg-info/ -pds_doi_service/core/.DS_Store -venv/ WhereAmI.py -build/ doi.db output/ transaction_history/ -.eggs/ docs/Makefile docs/design/PDS-technical-architecture.pdf docs/make.bat doi_temp.db tests/aaDOI_production_submitted_labels.zip tests/aaDOI_production_submitted_labels/ - - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..f6568a9a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: debug-statements + - id: check-yaml + files: .*\.(yaml|yml)$ + +- repo: https://github.com/asottile/reorder_python_imports + rev: v2.6.0 + hooks: + - id: reorder-python-imports + files: ^src/|tests/ + +- repo: https://github.com/pre-commit/mirrors-mypy.git + rev: v0.910 + hooks: + - id: mypy + files: ^src/|tests/ + +- repo: https://github.com/python/black + rev: 21.7b0 + hooks: + - id: black + files: ^src/|tests/ + +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + files: ^src/|tests/ + +- repo: local + hooks: + - id: tests + name: Tests + entry: pytest + language: system + stages: [push] + pass_filenames: false diff --git a/LICENSE.txt b/LICENSE.txt index a8d25871..816fe7be 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -199,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/NOTICE.txt b/NOTICE.txt index c3e4344d..778d59ed 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -28,4 +28,4 @@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 0b6612b2..9be5c08e 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # NASA PDS DOI Service -The PDS DOI Service provides tools for PDS operators to mint DOIs. +The PDS DOI Service provides tools for PDS operators to mint [DOI](https://www.doi.org/)s. ## Prerequisites - Python 3.7 or above - a login to OSTI server -## User Documentation +## User Documentation - https://nasa-pds.github.io/pds-doi-service/ + https://nasa-pds.github.io/pds-doi-service/ ## Developers @@ -17,24 +17,24 @@ Get the code and work on a branch git clone ... git checkout -b "#" - + Install virtual env pip install virtualenv python -m venv venv source venv/bin/activate - + Deploy dependencies: pip install -r requirements.txt pip install -r requirements_dev.txt - + or - + pip install -e . - + Update your local configuration to access the OSTI test server @@ -46,18 +46,18 @@ the following may be used as a template password = release_input_schematron = config/IAD3_scheematron.sch input_xsd = config/iad_schema.xsd - + [PDS4_DICTIONARY] url = https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_JSON_1D00.JSON pds_node_identifier = 0001_NASA_PDS_1.pds.Node.pds.name - + [LANDING_PAGES] # template url, arguments are # 1) product_class suffix, after _ # 2) lid # 3) vid url = https://pds.nasa.gov/ds-view/pds/view{}.jsp?identifier={}&version={} - + [OTHER] doi_publisher = NASA Planetary Data System global_keyword_values = PDS; PDS4; @@ -77,12 +77,12 @@ the following may be used as a template pds_registration_doi_token = 10.17189 logging_level=DEBUG - + ## Launch API server $ pip install pds-doi-service $ pds-doi-api - + The started service documentation is available on http://localhost:8080/PDS_APIs/pds_doi_api/0.1/ui/ ## Running with Docker @@ -109,7 +109,7 @@ This will launch the DOI Service container using the top-level `docker-compose.y specifies that environment variables be imported from `doi_service.env`. Modify `doi_service.env` to define any configuration values to override when the service is launched. -## Test +## Test ### Unit tests (for developers) : @@ -125,9 +125,9 @@ Note this will download reference test data. If they need to be updated you have You can also run them for a nicer reporting: - behave -f allure_behave.formatter:AllureFormatter -o ./allure ./features + behave -f allure_behave.formatter:AllureFormatter -o ./allure ./features allure service allure - + #### To report to testrail Test reports can be pushed to testrail: https://cae-testrail.jpl.nasa.gov/testrail/ @@ -139,13 +139,13 @@ Set you environment: export TESTRAIL_USER= export TESTRAIL_KEY= - + Run the tests: behave - + See the results in https://cae-testrail.jpl.nasa.gov/testrail/index.php?/projects/overview/168 - + ## Documentation management ### Design : @@ -161,7 +161,7 @@ Managed with sphinx brew install sphinx-doc pip install -r requirements_dev.txt cd docs - sphinx-build -b html source build -a + sphinx-build -b html source build -a ## Build & Release diff --git a/SCOPE.md b/SCOPE.md index e0329552..bc2f6f08 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -37,4 +37,4 @@ The system perform the reserve, draft, release, deactivate operations on OSTI sy - discipline node user - OSTI and dataCite, initially DOI are recorded by submission to OSTI (see https://www.osti.gov/iad2/docs) which submits them to DataCite - engineering node operator -- deployment platform (on-prem JPL) \ No newline at end of file +- deployment platform (on-prem JPL) diff --git a/docs/design/pds-doi-service-srd.md b/docs/design/pds-doi-service-srd.md index 1c6dbd0b..0f79fb31 100644 --- a/docs/design/pds-doi-service-srd.md +++ b/docs/design/pds-doi-service-srd.md @@ -40,7 +40,7 @@ The PDS Data Object Identifier (DOI) service is responsible for the management o 1. PDS EN operator receives email on DOIs which: - have been pending for too long (e.g. 2 days) - have been in a reserved or draft status without release for too long (e.g. 2 years) - + 2. the PDS EN operator can see a dashboard of all the DOIs with status 3. the submitter can see a summary and status of all the DOI submitted by its node. @@ -104,7 +104,7 @@ Besides a service will automatically raise alerts on DOIs which have been for to #### Reserve a DOI -TBD +TBD --- #### Draft a DOI @@ -162,8 +162,8 @@ These records are saved in a file directory structure <submitting discipline - input.<[xml|xlsx|csv]> - output.xml - comment (optional) - - + + These databases will be handled locally and later synchronized with the tracking service. Every command line operation will interact and feed these databases locally. @@ -217,5 +217,3 @@ TBD © 2020 California Institute of Technology. Government sponsorship acknowledged. - - diff --git a/docs/operations/OSTI.md b/docs/operations/OSTI.md index b57d1a3e..18e33a23 100644 --- a/docs/operations/OSTI.md +++ b/docs/operations/OSTI.md @@ -3,28 +3,28 @@ There are two methods: 1. OSTI web page 2. curl - + ## To submit record(s) via web page API in either XML or JSON format - + 1. In a browser enter: https://www.osti.gov/iad2test 2. click on Upload Request 3. click on Browse to locate file to be submitted -- enter credentials 4. success or failure ? - - + + ### To submit record to test server in XML format - + curl -u : https://www.osti.gov/iad2test/api/records -X POST -H "Content-Type: application/xml" -H "Accept: application/xml" --data @ATMOS_LADEE_NMS_Bundle_DOI_label_20180911.xml - + ### To submit record to test server in JSON format - + curl -u : https://www.osti.gov/iad2test/api/records -X POST -H "Content-Type: application/json" -H "Accept: application/json" --data @ATMOS_LADEE_NMS_Bundle_DOI_label_20180911. Json - -Curl will return either success or failure. An email will sent with the status. - -# Email + +Curl will return either success or failure. An email will sent with the status. + +# Email The emails related to test server submission are sent to pdsen-doi-test@jpl.nasa.gov @@ -34,7 +34,7 @@ Both email lists are managed as jpl groups, see https://dir.jpl.nasa.gov/groups/ The contact at OSTI is: -**Contact:** +**Contact:** Stephanie Gerics Office of Scientific and Technical Information (OSTI) diff --git a/docs/requirements/0.0.6-dev/REQUIREMENTS.md b/docs/requirements/0.0.6-dev/REQUIREMENTS.md index 06fff3ce..f5808697 100644 --- a/docs/requirements/0.0.6-dev/REQUIREMENTS.md +++ b/docs/requirements/0.0.6-dev/REQUIREMENTS.md @@ -4,33 +4,33 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version @@ -49,25 +49,25 @@ The enhancements which impact this requirements are: # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/requirements/0.0.7-dev/REQUIREMENTS.md b/docs/requirements/0.0.7-dev/REQUIREMENTS.md index 06fff3ce..f5808697 100644 --- a/docs/requirements/0.0.7-dev/REQUIREMENTS.md +++ b/docs/requirements/0.0.7-dev/REQUIREMENTS.md @@ -4,33 +4,33 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version @@ -49,25 +49,25 @@ The enhancements which impact this requirements are: # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/requirements/0.0.8-dev/REQUIREMENTS.md b/docs/requirements/0.0.8-dev/REQUIREMENTS.md index f1cf31eb..8f431a87 100644 --- a/docs/requirements/0.0.8-dev/REQUIREMENTS.md +++ b/docs/requirements/0.0.8-dev/REQUIREMENTS.md @@ -4,65 +4,65 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version -## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) +## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) This requirement is not impacted by the current version -## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) +## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) This requirement is not impacted by the current version # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/requirements/1.0.0/REQUIREMENTS.md b/docs/requirements/1.0.0/REQUIREMENTS.md index f1cf31eb..8f431a87 100644 --- a/docs/requirements/1.0.0/REQUIREMENTS.md +++ b/docs/requirements/1.0.0/REQUIREMENTS.md @@ -4,65 +4,65 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version -## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) +## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) This requirement is not impacted by the current version -## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) +## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) This requirement is not impacted by the current version # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/requirements/v1.2.0-dev/REQUIREMENTS.md b/docs/requirements/v1.2.0-dev/REQUIREMENTS.md index b4c3f951..80a5a1bb 100644 --- a/docs/requirements/v1.2.0-dev/REQUIREMENTS.md +++ b/docs/requirements/v1.2.0-dev/REQUIREMENTS.md @@ -4,167 +4,167 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version -## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) +## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) This requirement is not impacted by the current version -## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) +## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) This requirement is not impacted by the current version # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) This requirement is not impacted by the current version # default -## As a node operator, I want to include a DOI as a related identifier in the DOI metadata for parent PDS4 products. ([#69](https://github.com/NASA-PDS/pds-doi-service/issues/69)) +## As a node operator, I want to include a DOI as a related identifier in the DOI metadata for parent PDS4 products. ([#69](https://github.com/NASA-PDS/pds-doi-service/issues/69)) This requirement is not impacted by the current version -## As a node user, I want the LIDVID associated with a DOI to be automatically updated for accumulating bundles / collections. ([#97](https://github.com/NASA-PDS/pds-doi-service/issues/97)) +## As a node user, I want the LIDVID associated with a DOI to be automatically updated for accumulating bundles / collections. ([#97](https://github.com/NASA-PDS/pds-doi-service/issues/97)) This requirement is not impacted by the current version -## As the PDS, I want to mint DOIs through DataCite ([#103](https://github.com/NASA-PDS/pds-doi-service/issues/103)) +## As the PDS, I want to mint DOIs through DataCite ([#103](https://github.com/NASA-PDS/pds-doi-service/issues/103)) This requirement is not impacted by the current version -## Develop DOI Service - Registry integration component for accumulating bundles ([#111](https://github.com/NASA-PDS/pds-doi-service/issues/111)) +## Develop DOI Service - Registry integration component for accumulating bundles ([#111](https://github.com/NASA-PDS/pds-doi-service/issues/111)) This requirement is not impacted by the current version -## As a user, I want to see the lidvid of my DOIs in the email report ([#167](https://github.com/NASA-PDS/pds-doi-service/issues/167)) +## As a user, I want to see the lidvid of my DOIs in the email report ([#167](https://github.com/NASA-PDS/pds-doi-service/issues/167)) This requirement is not impacted by the current version -## As an operator, I want to reserve a DOI through DataCite ([#171](https://github.com/NASA-PDS/pds-doi-service/issues/171)) +## As an operator, I want to reserve a DOI through DataCite ([#171](https://github.com/NASA-PDS/pds-doi-service/issues/171)) This requirement is not impacted by the current version -## As an operator, I want query for one or more minted DOIs from DataCite ([#172](https://github.com/NASA-PDS/pds-doi-service/issues/172)) +## As an operator, I want query for one or more minted DOIs from DataCite ([#172](https://github.com/NASA-PDS/pds-doi-service/issues/172)) This requirement is not impacted by the current version -## As an operator, I want to query for a DOI's change history through DataCite ([#173](https://github.com/NASA-PDS/pds-doi-service/issues/173)) +## As an operator, I want to query for a DOI's change history through DataCite ([#173](https://github.com/NASA-PDS/pds-doi-service/issues/173)) This requirement is not impacted by the current version -## As an operator, I want to release a DOI through DataCite ([#174](https://github.com/NASA-PDS/pds-doi-service/issues/174)) +## As an operator, I want to release a DOI through DataCite ([#174](https://github.com/NASA-PDS/pds-doi-service/issues/174)) This requirement is not impacted by the current version -## As an operator, I want to update DOI metadata through DataCite ([#175](https://github.com/NASA-PDS/pds-doi-service/issues/175)) +## As an operator, I want to update DOI metadata through DataCite ([#175](https://github.com/NASA-PDS/pds-doi-service/issues/175)) This requirement is not impacted by the current version -## As an API user, I want to have pagination that is consistent with the PDS API. ([#176](https://github.com/NASA-PDS/pds-doi-service/issues/176)) +## As an API user, I want to have pagination that is consistent with the PDS API. ([#176](https://github.com/NASA-PDS/pds-doi-service/issues/176)) This requirement is not impacted by the current version -## As an API user I want to filter on lidvids with wildcards ([#177](https://github.com/NASA-PDS/pds-doi-service/issues/177)) +## As an API user I want to filter on lidvids with wildcards ([#177](https://github.com/NASA-PDS/pds-doi-service/issues/177)) This requirement is not impacted by the current version -## As an API user I want to filter on PDS3 Data Set IDs with wildcards ([#180](https://github.com/NASA-PDS/pds-doi-service/issues/180)) +## As an API user I want to filter on PDS3 Data Set IDs with wildcards ([#180](https://github.com/NASA-PDS/pds-doi-service/issues/180)) This requirement is not impacted by the current version -## As a user of the API, I want to see the DOI's title when I go GET /dois request ([#183](https://github.com/NASA-PDS/pds-doi-service/issues/183)) +## As a user of the API, I want to see the DOI's title when I go GET /dois request ([#183](https://github.com/NASA-PDS/pds-doi-service/issues/183)) This requirement is not impacted by the current version -## As an API user, I want to always have an update date for the DOIs ([#184](https://github.com/NASA-PDS/pds-doi-service/issues/184)) +## As an API user, I want to always have an update date for the DOIs ([#184](https://github.com/NASA-PDS/pds-doi-service/issues/184)) This requirement is not impacted by the current version -## As a SA, I want the operational deployment of the service to be secure ([#187](https://github.com/NASA-PDS/pds-doi-service/issues/187)) +## As a SA, I want the operational deployment of the service to be secure ([#187](https://github.com/NASA-PDS/pds-doi-service/issues/187)) This requirement is not impacted by the current version -## As a user, I want the application to support the history of PDS's DOIs, especially the one created for PDS3 products ([#192](https://github.com/NASA-PDS/pds-doi-service/issues/192)) +## As a user, I want the application to support the history of PDS's DOIs, especially the one created for PDS3 products ([#192](https://github.com/NASA-PDS/pds-doi-service/issues/192)) This requirement is not impacted by the current version -## As a system administrator, I want to be able to deploy pds_doi_service with python 3.6 ([#197](https://github.com/NASA-PDS/pds-doi-service/issues/197)) +## As a system administrator, I want to be able to deploy pds_doi_service with python 3.6 ([#197](https://github.com/NASA-PDS/pds-doi-service/issues/197)) This requirement is not impacted by the current version -## As a user, I want to use the API with ids containing a slash (/) ([#198](https://github.com/NASA-PDS/pds-doi-service/issues/198)) +## As a user, I want to use the API with ids containing a slash (/) ([#198](https://github.com/NASA-PDS/pds-doi-service/issues/198)) This requirement is not impacted by the current version -## As an operator, I want to know what version of the software I am running ([#200](https://github.com/NASA-PDS/pds-doi-service/issues/200)) +## As an operator, I want to know what version of the software I am running ([#200](https://github.com/NASA-PDS/pds-doi-service/issues/200)) This requirement is not impacted by the current version -## As an operator, I want to know how to deploy and use the API from the Sphinx documentation ([#201](https://github.com/NASA-PDS/pds-doi-service/issues/201)) +## As an operator, I want to know how to deploy and use the API from the Sphinx documentation ([#201](https://github.com/NASA-PDS/pds-doi-service/issues/201)) This requirement is not impacted by the current version -## As an operator, I want one place to go for all DOI Service / API / UI documentation ([#202](https://github.com/NASA-PDS/pds-doi-service/issues/202)) +## As an operator, I want one place to go for all DOI Service / API / UI documentation ([#202](https://github.com/NASA-PDS/pds-doi-service/issues/202)) This requirement is not impacted by the current version -## As a user of the command line, I don't want to see all the log info in the stdout ([#206](https://github.com/NASA-PDS/pds-doi-service/issues/206)) +## As a user of the command line, I don't want to see all the log info in the stdout ([#206](https://github.com/NASA-PDS/pds-doi-service/issues/206)) This requirement is not impacted by the current version -## As a command-line user, I want to be suggested to use -f option when a Warning exception is raised ([#207](https://github.com/NASA-PDS/pds-doi-service/issues/207)) +## As a command-line user, I want to be suggested to use -f option when a Warning exception is raised ([#207](https://github.com/NASA-PDS/pds-doi-service/issues/207)) This requirement is not impacted by the current version -## As a user, I want to run the commandline on windows ([#212](https://github.com/NASA-PDS/pds-doi-service/issues/212)) +## As a user, I want to run the commandline on windows ([#212](https://github.com/NASA-PDS/pds-doi-service/issues/212)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/requirements/v21.2.0/REQUIREMENTS.md b/docs/requirements/v21.2.0/REQUIREMENTS.md index b4c3f951..80a5a1bb 100644 --- a/docs/requirements/v21.2.0/REQUIREMENTS.md +++ b/docs/requirements/v21.2.0/REQUIREMENTS.md @@ -4,167 +4,167 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version -## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) +## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) This requirement is not impacted by the current version -## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) +## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) This requirement is not impacted by the current version # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) This requirement is not impacted by the current version # default -## As a node operator, I want to include a DOI as a related identifier in the DOI metadata for parent PDS4 products. ([#69](https://github.com/NASA-PDS/pds-doi-service/issues/69)) +## As a node operator, I want to include a DOI as a related identifier in the DOI metadata for parent PDS4 products. ([#69](https://github.com/NASA-PDS/pds-doi-service/issues/69)) This requirement is not impacted by the current version -## As a node user, I want the LIDVID associated with a DOI to be automatically updated for accumulating bundles / collections. ([#97](https://github.com/NASA-PDS/pds-doi-service/issues/97)) +## As a node user, I want the LIDVID associated with a DOI to be automatically updated for accumulating bundles / collections. ([#97](https://github.com/NASA-PDS/pds-doi-service/issues/97)) This requirement is not impacted by the current version -## As the PDS, I want to mint DOIs through DataCite ([#103](https://github.com/NASA-PDS/pds-doi-service/issues/103)) +## As the PDS, I want to mint DOIs through DataCite ([#103](https://github.com/NASA-PDS/pds-doi-service/issues/103)) This requirement is not impacted by the current version -## Develop DOI Service - Registry integration component for accumulating bundles ([#111](https://github.com/NASA-PDS/pds-doi-service/issues/111)) +## Develop DOI Service - Registry integration component for accumulating bundles ([#111](https://github.com/NASA-PDS/pds-doi-service/issues/111)) This requirement is not impacted by the current version -## As a user, I want to see the lidvid of my DOIs in the email report ([#167](https://github.com/NASA-PDS/pds-doi-service/issues/167)) +## As a user, I want to see the lidvid of my DOIs in the email report ([#167](https://github.com/NASA-PDS/pds-doi-service/issues/167)) This requirement is not impacted by the current version -## As an operator, I want to reserve a DOI through DataCite ([#171](https://github.com/NASA-PDS/pds-doi-service/issues/171)) +## As an operator, I want to reserve a DOI through DataCite ([#171](https://github.com/NASA-PDS/pds-doi-service/issues/171)) This requirement is not impacted by the current version -## As an operator, I want query for one or more minted DOIs from DataCite ([#172](https://github.com/NASA-PDS/pds-doi-service/issues/172)) +## As an operator, I want query for one or more minted DOIs from DataCite ([#172](https://github.com/NASA-PDS/pds-doi-service/issues/172)) This requirement is not impacted by the current version -## As an operator, I want to query for a DOI's change history through DataCite ([#173](https://github.com/NASA-PDS/pds-doi-service/issues/173)) +## As an operator, I want to query for a DOI's change history through DataCite ([#173](https://github.com/NASA-PDS/pds-doi-service/issues/173)) This requirement is not impacted by the current version -## As an operator, I want to release a DOI through DataCite ([#174](https://github.com/NASA-PDS/pds-doi-service/issues/174)) +## As an operator, I want to release a DOI through DataCite ([#174](https://github.com/NASA-PDS/pds-doi-service/issues/174)) This requirement is not impacted by the current version -## As an operator, I want to update DOI metadata through DataCite ([#175](https://github.com/NASA-PDS/pds-doi-service/issues/175)) +## As an operator, I want to update DOI metadata through DataCite ([#175](https://github.com/NASA-PDS/pds-doi-service/issues/175)) This requirement is not impacted by the current version -## As an API user, I want to have pagination that is consistent with the PDS API. ([#176](https://github.com/NASA-PDS/pds-doi-service/issues/176)) +## As an API user, I want to have pagination that is consistent with the PDS API. ([#176](https://github.com/NASA-PDS/pds-doi-service/issues/176)) This requirement is not impacted by the current version -## As an API user I want to filter on lidvids with wildcards ([#177](https://github.com/NASA-PDS/pds-doi-service/issues/177)) +## As an API user I want to filter on lidvids with wildcards ([#177](https://github.com/NASA-PDS/pds-doi-service/issues/177)) This requirement is not impacted by the current version -## As an API user I want to filter on PDS3 Data Set IDs with wildcards ([#180](https://github.com/NASA-PDS/pds-doi-service/issues/180)) +## As an API user I want to filter on PDS3 Data Set IDs with wildcards ([#180](https://github.com/NASA-PDS/pds-doi-service/issues/180)) This requirement is not impacted by the current version -## As a user of the API, I want to see the DOI's title when I go GET /dois request ([#183](https://github.com/NASA-PDS/pds-doi-service/issues/183)) +## As a user of the API, I want to see the DOI's title when I go GET /dois request ([#183](https://github.com/NASA-PDS/pds-doi-service/issues/183)) This requirement is not impacted by the current version -## As an API user, I want to always have an update date for the DOIs ([#184](https://github.com/NASA-PDS/pds-doi-service/issues/184)) +## As an API user, I want to always have an update date for the DOIs ([#184](https://github.com/NASA-PDS/pds-doi-service/issues/184)) This requirement is not impacted by the current version -## As a SA, I want the operational deployment of the service to be secure ([#187](https://github.com/NASA-PDS/pds-doi-service/issues/187)) +## As a SA, I want the operational deployment of the service to be secure ([#187](https://github.com/NASA-PDS/pds-doi-service/issues/187)) This requirement is not impacted by the current version -## As a user, I want the application to support the history of PDS's DOIs, especially the one created for PDS3 products ([#192](https://github.com/NASA-PDS/pds-doi-service/issues/192)) +## As a user, I want the application to support the history of PDS's DOIs, especially the one created for PDS3 products ([#192](https://github.com/NASA-PDS/pds-doi-service/issues/192)) This requirement is not impacted by the current version -## As a system administrator, I want to be able to deploy pds_doi_service with python 3.6 ([#197](https://github.com/NASA-PDS/pds-doi-service/issues/197)) +## As a system administrator, I want to be able to deploy pds_doi_service with python 3.6 ([#197](https://github.com/NASA-PDS/pds-doi-service/issues/197)) This requirement is not impacted by the current version -## As a user, I want to use the API with ids containing a slash (/) ([#198](https://github.com/NASA-PDS/pds-doi-service/issues/198)) +## As a user, I want to use the API with ids containing a slash (/) ([#198](https://github.com/NASA-PDS/pds-doi-service/issues/198)) This requirement is not impacted by the current version -## As an operator, I want to know what version of the software I am running ([#200](https://github.com/NASA-PDS/pds-doi-service/issues/200)) +## As an operator, I want to know what version of the software I am running ([#200](https://github.com/NASA-PDS/pds-doi-service/issues/200)) This requirement is not impacted by the current version -## As an operator, I want to know how to deploy and use the API from the Sphinx documentation ([#201](https://github.com/NASA-PDS/pds-doi-service/issues/201)) +## As an operator, I want to know how to deploy and use the API from the Sphinx documentation ([#201](https://github.com/NASA-PDS/pds-doi-service/issues/201)) This requirement is not impacted by the current version -## As an operator, I want one place to go for all DOI Service / API / UI documentation ([#202](https://github.com/NASA-PDS/pds-doi-service/issues/202)) +## As an operator, I want one place to go for all DOI Service / API / UI documentation ([#202](https://github.com/NASA-PDS/pds-doi-service/issues/202)) This requirement is not impacted by the current version -## As a user of the command line, I don't want to see all the log info in the stdout ([#206](https://github.com/NASA-PDS/pds-doi-service/issues/206)) +## As a user of the command line, I don't want to see all the log info in the stdout ([#206](https://github.com/NASA-PDS/pds-doi-service/issues/206)) This requirement is not impacted by the current version -## As a command-line user, I want to be suggested to use -f option when a Warning exception is raised ([#207](https://github.com/NASA-PDS/pds-doi-service/issues/207)) +## As a command-line user, I want to be suggested to use -f option when a Warning exception is raised ([#207](https://github.com/NASA-PDS/pds-doi-service/issues/207)) This requirement is not impacted by the current version -## As a user, I want to run the commandline on windows ([#212](https://github.com/NASA-PDS/pds-doi-service/issues/212)) +## As a user, I want to run the commandline on windows ([#212](https://github.com/NASA-PDS/pds-doi-service/issues/212)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/requirements/v9.8.7/REQUIREMENTS.md b/docs/requirements/v9.8.7/REQUIREMENTS.md index d4c47ece..da918814 100644 --- a/docs/requirements/v9.8.7/REQUIREMENTS.md +++ b/docs/requirements/v9.8.7/REQUIREMENTS.md @@ -4,123 +4,123 @@ Requirements Summary # DOI management -## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) +## The software shall be capable of accepting a request to create a draft DOI. ([#5](https://github.com/NASA-PDS/pds-doi-service/issues/5)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) +## The software shall be capable of accepting a request to reserve a DOI. ([#6](https://github.com/NASA-PDS/pds-doi-service/issues/6)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) +## The software shall be capable of accepting a request to release a DOI. ([#7](https://github.com/NASA-PDS/pds-doi-service/issues/7)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) +## The software shall be capable of accepting a request to deactivate a DOI. ([#8](https://github.com/NASA-PDS/pds-doi-service/issues/8)) This requirement is not impacted by the current version -## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) +## The software shall be capable of accepting a request to update DOI metadata. ([#9](https://github.com/NASA-PDS/pds-doi-service/issues/9)) This requirement is not impacted by the current version -## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) +## The software shall be capable of batch processing >1 DOI requests. ([#10](https://github.com/NASA-PDS/pds-doi-service/issues/10)) This requirement is not impacted by the current version # DOI metadata -## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) +## The software shall be capable of autonomously generating the minimum set of DOI metadata from PDS4 Collection, Bundle, Document products. ([#11](https://github.com/NASA-PDS/pds-doi-service/issues/11)) This requirement is not impacted by the current version -## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) +## The software shall validate a minimum set of metadata is provided when reserving, releasing, or updating a DOI. This minimum set of metadata will be defined by the PDS DOI Working Group. ([#12](https://github.com/NASA-PDS/pds-doi-service/issues/12)) This requirement is not impacted by the current version -## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) +## The software shall validate the DOI metadata when reserving, releasing, or updating a DOI. ([#13](https://github.com/NASA-PDS/pds-doi-service/issues/13)) This requirement is not impacted by the current version # DOI interface support -## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) +## The software shall maintain a database of PDS DOIs and their current state. ([#14](https://github.com/NASA-PDS/pds-doi-service/issues/14)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) +## The software shall maintain the ability to manage DOIs through OSTI ([#15](https://github.com/NASA-PDS/pds-doi-service/issues/15)) This requirement is not impacted by the current version -## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) +## The software shall maintain the ability to manage DOIs through DataCite. ([#16](https://github.com/NASA-PDS/pds-doi-service/issues/16)) This requirement is not impacted by the current version # DOI-management -## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) +## The software shall provide a Status capability that will allow a user to query for the current status of a DOI ([#30](https://github.com/NASA-PDS/pds-doi-service/issues/30)) This requirement is not impacted by the current version -## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) +## The software shall provide the capability of producing a DOI Status Report based upon a user-specified query ([#35](https://github.com/NASA-PDS/pds-doi-service/issues/35)) This requirement is not impacted by the current version # default -## As a node operator, I want to include a DOI as a related identifier in the DOI metadata for parent PDS4 products. ([#69](https://github.com/NASA-PDS/pds-doi-service/issues/69)) +## As a node operator, I want to include a DOI as a related identifier in the DOI metadata for parent PDS4 products. ([#69](https://github.com/NASA-PDS/pds-doi-service/issues/69)) This requirement is not impacted by the current version -## As a node user, I want the LIDVID associated with a DOI to be automatically updated for accumulating bundles / collections. ([#97](https://github.com/NASA-PDS/pds-doi-service/issues/97)) +## As a node user, I want the LIDVID associated with a DOI to be automatically updated for accumulating bundles / collections. ([#97](https://github.com/NASA-PDS/pds-doi-service/issues/97)) This requirement is not impacted by the current version -## As the PDS, I want to mint DOIs through DataCite ([#103](https://github.com/NASA-PDS/pds-doi-service/issues/103)) +## As the PDS, I want to mint DOIs through DataCite ([#103](https://github.com/NASA-PDS/pds-doi-service/issues/103)) This requirement is not impacted by the current version -## Develop DOI Service - Registry integration component for accumulating bundles ([#111](https://github.com/NASA-PDS/pds-doi-service/issues/111)) +## Develop DOI Service - Registry integration component for accumulating bundles ([#111](https://github.com/NASA-PDS/pds-doi-service/issues/111)) This requirement is not impacted by the current version -## As an operator, I want to reserve a DOI through DataCite ([#171](https://github.com/NASA-PDS/pds-doi-service/issues/171)) +## As an operator, I want to reserve a DOI through DataCite ([#171](https://github.com/NASA-PDS/pds-doi-service/issues/171)) This requirement is not impacted by the current version -## As an operator, I want query for one or more minted DOIs from DataCite ([#172](https://github.com/NASA-PDS/pds-doi-service/issues/172)) +## As an operator, I want query for one or more minted DOIs from DataCite ([#172](https://github.com/NASA-PDS/pds-doi-service/issues/172)) This requirement is not impacted by the current version -## As an operator, I want to query for a DOI's change history through DataCite ([#173](https://github.com/NASA-PDS/pds-doi-service/issues/173)) +## As an operator, I want to query for a DOI's change history through DataCite ([#173](https://github.com/NASA-PDS/pds-doi-service/issues/173)) This requirement is not impacted by the current version -## As an operator, I want to release a DOI through DataCite ([#174](https://github.com/NASA-PDS/pds-doi-service/issues/174)) +## As an operator, I want to release a DOI through DataCite ([#174](https://github.com/NASA-PDS/pds-doi-service/issues/174)) This requirement is not impacted by the current version -## As an operator, I want to update DOI metadata through DataCite ([#175](https://github.com/NASA-PDS/pds-doi-service/issues/175)) +## As an operator, I want to update DOI metadata through DataCite ([#175](https://github.com/NASA-PDS/pds-doi-service/issues/175)) This requirement is not impacted by the current version -## As an API user, I want to have pagination that is consistent with the PDS API. ([#176](https://github.com/NASA-PDS/pds-doi-service/issues/176)) +## As an API user, I want to have pagination that is consistent with the PDS API. ([#176](https://github.com/NASA-PDS/pds-doi-service/issues/176)) This requirement is not impacted by the current version -## As an API user I want to filter on lidvids with wildcards ([#177](https://github.com/NASA-PDS/pds-doi-service/issues/177)) +## As an API user I want to filter on lidvids with wildcards ([#177](https://github.com/NASA-PDS/pds-doi-service/issues/177)) This requirement is not impacted by the current version -## As an API user I want to filter on PDS3 Data Set IDs with wildcards ([#180](https://github.com/NASA-PDS/pds-doi-service/issues/180)) +## As an API user I want to filter on PDS3 Data Set IDs with wildcards ([#180](https://github.com/NASA-PDS/pds-doi-service/issues/180)) This requirement is not impacted by the current version -## As a user of the API, I want to see the DOI's title when I go GET /dois request ([#183](https://github.com/NASA-PDS/pds-doi-service/issues/183)) +## As a user of the API, I want to see the DOI's title when I go GET /dois request ([#183](https://github.com/NASA-PDS/pds-doi-service/issues/183)) This requirement is not impacted by the current version -## As an API user, I want to always have an update date for the DOIs ([#184](https://github.com/NASA-PDS/pds-doi-service/issues/184)) +## As an API user, I want to always have an update date for the DOIs ([#184](https://github.com/NASA-PDS/pds-doi-service/issues/184)) -This requirement is not impacted by the current version \ No newline at end of file +This requirement is not impacted by the current version diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 00000000..63ee6cc7 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,13 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} diff --git a/docs/source/conf.py b/docs/source/conf.py index d8760b96..02253803 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- project = 'PDS DOI Service' -copyright = '2020 California Institute of Technology' +copyright = '2020–2021 California Institute of Technology' author = 'NASA PDS Engineering Node' @@ -76,3 +76,8 @@ html_logo = '_static/images/PDS_Planets.png' +html_context = { + 'css_files': [ + '_static/theme_overrides.css', # override wide tables in RTD theme + ], +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 2184acc4..0fb2317b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,11 +1,16 @@ PDS DOI Service =============== -The PDS DOI Service enable management of DOI for PDS with the following operations: reserve, draft, release, deactivate. +The Planetary Data System (PDS_) Digital Object Identifier (DOI_) Service +enable management of DOIs for PDS with the following operations: reserve, +draft, release, deactivate. -The PDS DOIs are registered through the `OSTI`_ infrastructure. OSTI registers PDS DOI on `DataCite`_ infrastructure +The PDS DOIs are registered through the `OSTI`_ infrastructure. OSTI registers +PDS DOI on `DataCite`_ infrastructure. -Currently, this service can be deployed as a python package 'pds-doi-service' and is executed locally through a command line `pds-doi-cmd` or remotely through a web API server. +Currently, this service can be deployed as a python package +``pds-doi-service`` and is executed locally through a command line +``pds-doi-cmd`` or remotely through a web API server. .. toctree:: @@ -18,6 +23,7 @@ Currently, this service can be deployed as a python package 'pds-doi-service' an support/index -.. _Python: https://www.python.org/ .. _OSTI: https://www.osti.gov/data-services .. _dataCite: https://datacite.org +.. _DOI: https://doi.org/ +.. _PDS: https://pds.nasa.gov/ diff --git a/docs/source/installation/index.rst b/docs/source/installation/index.rst index 8504c85e..673b6300 100644 --- a/docs/source/installation/index.rst +++ b/docs/source/installation/index.rst @@ -17,8 +17,8 @@ requirements: --version`` to find out. Consult your operating system instructions or system administrator to install -the required packages. For those without system administrator access and are -feeling anxious, you could try a local (home directory) Python_ 3 installation +the required packages. For those without system administrator access and are +feeling anxious, you could try a local (home directory) Python_ 3 installation using a Miniconda_ installation. @@ -125,7 +125,7 @@ You can explore the API documentation and test it from its UI: http://localhost: Upgrade Software ---------------- -To check and install an upgrade to the software, run the following command in your +To check and install an upgrade to the software, run the following command in your virtual environment:: pip install --upgrade pds-doi-service diff --git a/docs/source/support/index.rst b/docs/source/support/index.rst index 41370afd..8c9cbb2c 100644 --- a/docs/source/support/index.rst +++ b/docs/source/support/index.rst @@ -9,4 +9,3 @@ contributions. Please submit new issues into our repository Github Issue Tracking: Issue Tracking: https://github.com/NASA-PDS/pds-doi-service/issues - diff --git a/docs/source/usage/index.rst b/docs/source/usage/index.rst index b90dbc18..64714b55 100644 --- a/docs/source/usage/index.rst +++ b/docs/source/usage/index.rst @@ -37,4 +37,3 @@ Usage Information .. _OSTI: https://www.osti.gov/data-services .. _dataCite: https://datacite.org - diff --git a/features/reserve_doi.feature b/features/reserve_doi.feature index 83bec73c..8f20c126 100644 --- a/features/reserve_doi.feature +++ b/features/reserve_doi.feature @@ -14,7 +14,7 @@ Feature: reserve a OSTI DOI Scenario Outline: Reserve an OSTI DOI with an invalid PDS4 label Given an invalid PDS4 label at When reserve DOI in OSTI format at - Then a reading error report is generated for + Then a reading error report is generated for Examples: Invalid PDS4 labels | input_type | node_value | input_value | error_report | | bundle | img | tests/data/invalid_bundle.xml | tests/data/reserve_error_report.txt | @@ -47,15 +47,3 @@ Feature: reserve a OSTI DOI # | aaDOI_production_submitted_labels | rms | RINGS_reserve_VIMS_20200406/aaaSubmitted_by_RINGS_reserve_20200406/DOI_RMS_Cassini-Reserved-2020-03-31_edited.xlsx | RINGS_reserve_VIMS_20200406/aaaRegistered_by_EN_reserve_20200316/DOI_reserved_all_records.xml | # This next doesn't compare due to "[Showalter #2], new:[Showalter #3]' # | aaDOI_production_submitted_labels | rms | RINGS_reserve_VIMS_20200406/aaaSubmitted_by_RINGS_reserve_20200406/DOI_RMS_Cassini-Reserved-2020-03-31_edited.xlsx | RINGS_reserve_VIMS_20200406/aaaRegistered_by_EN_reserve_20200316/DOI_reserved_cassini_vims_saturn_document_vims-browse-interpretation-key_edit.xml | - - - - - - - - - - - - diff --git a/features/steps/steps.py b/features/steps/steps.py index ffccfc14..fb7484e2 100644 --- a/features/steps/steps.py +++ b/features/steps/steps.py @@ -266,8 +266,3 @@ def step_lidvid_already_submitted_exception_is_raised(context): logger.info(f'expected message {excepted_exception_msg}') logger.info(f'found msg is {context.exception_msg}') assert context.exception_msg == excepted_exception_msg - - - - - diff --git a/input/DOI_Release_20210216_from_draft.json b/input/DOI_Release_20210216_from_draft.json index 26cade4b..3be5a5c6 100644 --- a/input/DOI_Release_20210216_from_draft.json +++ b/input/DOI_Release_20210216_from_draft.json @@ -39,4 +39,3 @@ "contact_phone": "818.393.7165" } ] - diff --git a/input/DOI_Reserved_GEO_200318.csv b/input/DOI_Reserved_GEO_200318.csv index 6c063b30..01cb8eaa 100644 --- a/input/DOI_Reserved_GEO_200318.csv +++ b/input/DOI_Reserved_GEO_200318.csv @@ -1,4 +1,4 @@ status,title,publication_date,product_type_specific,author_last_name,author_first_name,related_resource Reserved,Laboratory Shocked Feldspars Collection #1,3/11/20,PDS4 Collection,Johnson,J. R.,urn:nasa:pds:lab_shocked_feldspars::10.0 Reserved,Laboratory Shocked Feldspars Collection #2,3/12/20,PDS4 Collection,Johnson_2,J2. R2.,urn:nasa:pds:lab_shocked_feldspars_2::10.0 -Reserved,Laboratory Shocked Feldspars Collection #3,3/12/20,PDS4 Collection,Johnson_3,J3. R3.,urn:nasa:pds:lab_shocked_feldspars_3::10.0 \ No newline at end of file +Reserved,Laboratory Shocked Feldspars Collection #3,3/12/20,PDS4 Collection,Johnson_3,J3. R3.,urn:nasa:pds:lab_shocked_feldspars_3::10.0 diff --git a/input/DOI_Reserved_PDS3.csv b/input/DOI_Reserved_PDS3.csv index 95432d67..465676be 100644 --- a/input/DOI_Reserved_PDS3.csv +++ b/input/DOI_Reserved_PDS3.csv @@ -1,3 +1,2 @@ status,title,publication_date,product_type_specific,author_last_name,author_first_name,related_resource Reserved,LRO MOON MINI-RF 2/3/5 BISTATIC RADAR V3.0,12/13/2019,PDS3 Dataset,Santo,Andrew,LRO-L-MRFLRO-2/3/5-BISTATIC-V3.0 - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f8b799fd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ["setuptools >= 46.4.0", "wheel"] + +# uncomment to enable pep517 after versioneer problem is fixed. +# https://github.com/python-versioneer/python-versioneer/issues/193 +#build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 120 diff --git a/setup.cfg b/setup.cfg index 03409106..714a4848 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,156 @@ -# TODO: someday we should move everything from setup.py and into here. -# -# See https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html +[metadata] +name = pds-doi-service +author = PDS +author_email = pds_operator@jpl.nasa.gov +description = Digital Object Identifier service for the Planetary Data System +long_description = file: README.md +long_description_content_type = text/markdown +license = apache-2.0 +keywords = pds, doi, osti, dataCite +url = https://github.com/NASA-PDS/pds-doi-service +download_url = https://github.com/NASA-PDS/pds-doi-service/releases/ +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Web Environment + Framework :: Flask + Intended Audience :: Science/Research + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Programming Language :: Python :: 3 + Topic :: Scientific/Engineering + + +[options] +install_requires = + appdirs>=1.4 + beautifulsoup4==4.8.2 + bs4==0.0.1 + certifi==2020.4.5.1 + chardet==3.0.4 + connexion[swagger-ui] == 2.7.0 + dataclasses==0.7; python_version <= '3.6' + distlib==0.3.1 + filelock==3.0.12 + Flask==1.1.2 + flask-cors==3.0.9 + idna==2.9 + importlib-metadata==1.5.0 + importlib-resources==3.0.0 + jinja2==3.0.1 + jsonschema==3.0.0 + lxml>=4.5 + nltk==3.5 + numpy>=1.18 + openapi-schema-validator==0.1.4 + openpyxl==3.0 + pandas>=1.0 + pystache>=0.5 + python-dateutil>=2.8 + pytz==2020.1 + requests>=2.23 + six>=1.14 + soupsieve>=2.0 + urllib3>=1.25 + waitress==2.0.0 + Werkzeug==0.16.0 + wheel + xlrd>=1.2 + xmlschema==1.5.1 + xmltodict>=0.12 + zipp>=3.1 +zip_safe = False +include_package_data = True +package_dir = + = src +packages = find: +python_requires = >= 3.9 +test_suite = pds_doi_service.test.suite + + +[options.extras_require] +dev = + black + flake8 + flake8-bugbear + flake8-docstrings + pep8-naming + mypy + pydocstyle + coverage + pytest + pytest-cov + pytest-watch + pytest-xdist + pre-commit + sphinx + sphinx-rtd-theme + tox + flask_testing==0.8.0 + sphinx-rtd-theme==0.5.0 + sphinx-argparse==0.2.5 + behave==1.2.6 + allure-behave==2.8.13 + behave-testrail-reporter==0.4.0 + + +[options.entry_points] +console_scripts = + pds-doi-cmd=pds_doi_service.core.cmd.pds_doi_cmd:main + pds-doi-api=pds_doi_service.api.__main__:main + + +[options.packages.find] +where = src + + +[coverage:run] +omit = */_version.py,*/__init__.py + [test] +[tool:pytest] +addopts = -n auto --cov=pds + [install] +[flake8] +max-line-length = 120 +extend-exclude = versioneer.py,_version.py,docs,tests,setup.py +docstring_convention = google + +# Ignoring: +# E203 prevents flake8 from complaining about whitespace around slice +# components. Black formats per PEP8 and flake8 doesn't like some of +# this. +# +# E501 prevents flake8 from complaining line lengths > 79. We will use +# flake8-bugbear's B950 to handle line length lint errors. This trips +# when a line is > max-line-length + 10%. +extend-ignore = E203, E501 + +# Selects following test categories: +# D: Docstring errors and warnings +# E, W: PEP8 errors and warnings +# F: PyFlakes codes +# N: PEP8 Naming plugin codes +# B: flake8-bugbear codes +# B***: Specific flake8-bugbear opinionated warnings to trigger +# B902: Invalid first argument used for method. Use self for instance +# methods, and cls for class methods +# B903: Use collections.namedtuple (or typing.NamedTuple) for data classes +# that only set attributes in an __init__ method, and do nothing else. +# B950: Line too long. This is a pragmatic equivalent of pycodestyle's +# E501: it considers "max-line-length" but only triggers when the value +# has been exceeded by more than 10%. +select = D,E,F,N,W,B,B902,B903,B950 + +[mypy] + +[mypy-pds.*._version] +# We don't care about issues in versioneer's files +ignore_errors = True + [versioneer] VCS = git style = pep440 diff --git a/setup.py b/setup.py index 4301dd1a..2258a7f0 100644 --- a/setup.py +++ b/setup.py @@ -1,94 +1,9 @@ -# -*- coding: utf-8 -*- - -import setuptools, versioneer - -# Package Metadata -# ---------------- -# -# Normally we'd have a `pds` namespace package with `doi_service` inside of it, but when I try to do that -# I get errors like: -# -# connexion.exceptions.ResolverError: 'DoiRecord': + def from_dict(cls, dikt) -> "DoiRecord": """Returns the dict as a model :param dikt: A dict. diff --git a/src/pds_doi_service/api/models/doi_summary.py b/src/pds_doi_service/api/models/doi_summary.py index ef19cedf..1b31ac3f 100755 --- a/src/pds_doi_service/api/models/doi_summary.py +++ b/src/pds_doi_service/api/models/doi_summary.py @@ -1,19 +1,20 @@ # coding: utf-8 - from __future__ import absolute_import + from datetime import datetime # noqa: F401 -from pds_doi_service.api.models import Model from pds_doi_service.api import util +from pds_doi_service.api.models import Model class DoiSummary(Model): """ NOTE: This class was auto generated by the swagger code generator program. """ - def __init__(self, doi=None, identifier=None, title=None, - node=None, submitter=None, status=None, - update_date=None): # noqa: E501 + + def __init__( + self, doi=None, identifier=None, title=None, node=None, submitter=None, status=None, update_date=None + ): # noqa: E501 """DoiSummary - a model defined in Swagger :param doi: The doi of this DoiSummary. # noqa: E501 @@ -32,23 +33,23 @@ def __init__(self, doi=None, identifier=None, title=None, :type update_date: datetime """ self.swagger_types = { - 'doi': str, - 'identifier': str, - 'title': str, - 'node': str, - 'submitter': str, - 'status': str, - 'update_date': datetime + "doi": str, + "identifier": str, + "title": str, + "node": str, + "submitter": str, + "status": str, + "update_date": datetime, } self.attribute_map = { - 'doi': 'doi', - 'identifier': 'identifier', - 'title': 'title', - 'node': 'node', - 'submitter': 'submitter', - 'status': 'status', - 'update_date': 'update_date' + "doi": "doi", + "identifier": "identifier", + "title": "title", + "node": "node", + "submitter": "submitter", + "status": "status", + "update_date": "update_date", } self._doi = doi self._identifier = identifier @@ -59,7 +60,7 @@ def __init__(self, doi=None, identifier=None, title=None, self._update_date = update_date @classmethod - def from_dict(cls, dikt) -> 'DoiSummary': + def from_dict(cls, dikt) -> "DoiSummary": """Returns the dict as a model :param dikt: A dict. diff --git a/src/pds_doi_service/api/models/label_payload.py b/src/pds_doi_service/api/models/label_payload.py index a1c2bf59..aae8a880 100755 --- a/src/pds_doi_service/api/models/label_payload.py +++ b/src/pds_doi_service/api/models/label_payload.py @@ -1,21 +1,30 @@ # coding: utf-8 - from __future__ import absolute_import -from datetime import date, datetime # noqa: F401 -from typing import List, Dict # noqa: F401 +from datetime import date +from datetime import datetime +from typing import Dict +from typing import List -from pds_doi_service.api.models import Model from pds_doi_service.api import util +from pds_doi_service.api.models import Model class LabelPayload(Model): """ NOTE: This class was auto generated by the swagger code generator program. """ - def __init__(self, status=None, title= None, publication_date=None, - product_type_specific=None, author_last_name=None, - author_first_name=None, related_resource=None): # noqa: E501 + + def __init__( + self, + status=None, + title=None, + publication_date=None, + product_type_specific=None, + author_last_name=None, + author_first_name=None, + related_resource=None, + ): # noqa: E501 """LabelPayload - a model defined in Swagger :param status: The status of this LabelPayload. # noqa: E501 @@ -34,23 +43,23 @@ def __init__(self, status=None, title= None, publication_date=None, :type related_resource: str """ self.swagger_types = { - 'status': str, - 'title': str, - 'publication_date': datetime, - 'product_type_specific': str, - 'author_last_name': str, - 'author_first_name': str, - 'related_resource': str + "status": str, + "title": str, + "publication_date": datetime, + "product_type_specific": str, + "author_last_name": str, + "author_first_name": str, + "related_resource": str, } self.attribute_map = { - 'status': 'status', - 'title': 'title', - 'publication_date': 'publication_date', - 'product_type_specific': 'product_type_specific', - 'author_last_name': 'author_last_name', - 'author_first_name': 'author_first_name', - 'related_resource': 'related_resource' + "status": "status", + "title": "title", + "publication_date": "publication_date", + "product_type_specific": "product_type_specific", + "author_last_name": "author_last_name", + "author_first_name": "author_first_name", + "related_resource": "related_resource", } self._status = status self._title = title @@ -61,7 +70,7 @@ def __init__(self, status=None, title= None, publication_date=None, self._related_resource = related_resource @classmethod - def from_dict(cls, dikt) -> 'LabelPayload': + def from_dict(cls, dikt) -> "LabelPayload": """Returns the dict as a model :param dikt: A dict. diff --git a/src/pds_doi_service/api/models/labels_payload.py b/src/pds_doi_service/api/models/labels_payload.py index 271e9c06..e4a20ec4 100755 --- a/src/pds_doi_service/api/models/labels_payload.py +++ b/src/pds_doi_service/api/models/labels_payload.py @@ -1,36 +1,34 @@ # coding: utf-8 - from __future__ import absolute_import -from datetime import date, datetime # noqa: F401 -from typing import List, Dict # noqa: F401 +from datetime import date +from datetime import datetime +from typing import Dict +from typing import List +from pds_doi_service.api import util from pds_doi_service.api.models import Model from pds_doi_service.api.models.label_payload import LabelPayload # noqa: F401,E501 -from pds_doi_service.api import util class LabelsPayload(Model): """ NOTE: This class was auto generated by the swagger code generator program. """ + def __init__(self, labels=None): # noqa: E501 """LabelsPayload - a model defined in Swagger :param labels: The labels of this LabelsPayload. # noqa: E501 :type labels: List[LabelPayload] """ - self.swagger_types = { - 'labels': List[LabelPayload] - } + self.swagger_types = {"labels": List[LabelPayload]} - self.attribute_map = { - 'labels': 'labels' - } + self.attribute_map = {"labels": "labels"} self._labels = labels @classmethod - def from_dict(cls, dikt) -> 'LabelsPayload': + def from_dict(cls, dikt) -> "LabelsPayload": """Returns the dict as a model :param dikt: A dict. diff --git a/src/pds_doi_service/api/test/__init__.py b/src/pds_doi_service/api/test/__init__.py index 4842ee85..3d865a2a 100755 --- a/src/pds_doi_service/api/test/__init__.py +++ b/src/pds_doi_service/api/test/__init__.py @@ -1,11 +1,9 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for API -''' - - +""" import unittest + from . import test_dois_controller diff --git a/src/pds_doi_service/api/test/_base.py b/src/pds_doi_service/api/test/_base.py index 434c73aa..52ae7203 100644 --- a/src/pds_doi_service/api/test/_base.py +++ b/src/pds_doi_service/api/test/_base.py @@ -1,18 +1,15 @@ # encoding: utf-8 - """ Planetary Data System's Digital Object Identifier service — API testing base classes """ - +import logging from flask_testing import TestCase from pds_doi_service.api.__main__ import init_app -import logging class BaseTestCase(TestCase): - def create_app(self): - logging.getLogger('connexion.operation').setLevel('ERROR') + logging.getLogger("connexion.operation").setLevel("ERROR") app = init_app() return app.app diff --git a/src/pds_doi_service/api/test/data/datacite/reserve_record b/src/pds_doi_service/api/test/data/datacite/reserve_record index db68d417..3266cee3 100644 --- a/src/pds_doi_service/api/test/data/datacite/reserve_record +++ b/src/pds_doi_service/api/test/data/datacite/reserve_record @@ -107,4 +107,4 @@ "language": "en" } } -} \ No newline at end of file +} diff --git a/src/pds_doi_service/api/test/data/osti/draft_record b/src/pds_doi_service/api/test/data/osti/draft_record index 29374edc..fdb09630 100644 --- a/src/pds_doi_service/api/test/data/osti/draft_record +++ b/src/pds_doi_service/api/test/data/osti/draft_record @@ -47,4 +47,3 @@ 818.393.7165 - diff --git a/src/pds_doi_service/api/test/data/osti/output.xml b/src/pds_doi_service/api/test/data/osti/output.xml index 15ad19f6..39b645e1 100644 --- a/src/pds_doi_service/api/test/data/osti/output.xml +++ b/src/pds_doi_service/api/test/data/osti/output.xml @@ -124,4 +124,3 @@ 818.393.7165 - diff --git a/src/pds_doi_service/api/test/data/osti/release_record b/src/pds_doi_service/api/test/data/osti/release_record index 4bb8e77e..caaf5391 100644 --- a/src/pds_doi_service/api/test/data/osti/release_record +++ b/src/pds_doi_service/api/test/data/osti/release_record @@ -47,4 +47,3 @@ 818.393.7165 - diff --git a/src/pds_doi_service/api/test/data/osti/reserve_record b/src/pds_doi_service/api/test/data/osti/reserve_record index 4e6724bd..3193369c 100644 --- a/src/pds_doi_service/api/test/data/osti/reserve_record +++ b/src/pds_doi_service/api/test/data/osti/reserve_record @@ -63,4 +63,3 @@ "contact_phone": "818.393.7165" } ] - diff --git a/src/pds_doi_service/api/test/test_dois_controller.py b/src/pds_doi_service/api/test/test_dois_controller.py index b9e6502c..f5c52631 100755 --- a/src/pds_doi_service/api/test/test_dois_controller.py +++ b/src/pds_doi_service/api/test/test_dois_controller.py @@ -1,32 +1,36 @@ # coding: utf-8 - from __future__ import absolute_import -from datetime import datetime import json import os -from os.path import abspath, exists, join import unittest +from datetime import datetime +from os.path import abspath +from os.path import exists +from os.path import join from unittest.mock import patch -from pkg_resources import resource_filename - import pds_doi_service.api.controllers.dois_controller import pds_doi_service.core.outputs.osti.osti_web_client import pds_doi_service.core.outputs.transaction - -from ._base import BaseTestCase from pds_doi_service.api.encoder import JSONEncoder -from pds_doi_service.api.models import (DoiRecord, DoiSummary, - LabelsPayload, LabelPayload) +from pds_doi_service.api.models import DoiRecord +from pds_doi_service.api.models import DoiSummary +from pds_doi_service.api.models import LabelPayload +from pds_doi_service.api.models import LabelsPayload from pds_doi_service.core.entities.doi import DoiStatus from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE from pds_doi_service.core.util.config_parser import DOIConfigUtil -from pds_doi_service.core.outputs.service import DOIServiceFactory, SERVICE_TYPE_DATACITE +from pkg_resources import resource_filename + +from ._base import BaseTestCase class TestDoisController(BaseTestCase): """DoisController integration test stubs""" + # These attributes are defined at class level so it may be accessed # by patched methods test_data_dir = None @@ -35,24 +39,22 @@ class TestDoisController(BaseTestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.test_data_dir = join(cls.test_dir, 'data') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + cls.test_dir = resource_filename(__name__, "") + cls.test_data_dir = join(cls.test_dir, "data") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) cls.service_type = DOIServiceFactory.get_service_type() # Path to a temporary database to re-instantiate for every test - cls.temp_db = join(cls.test_data_dir, 'temp.db') + cls.temp_db = join(cls.test_data_dir, "temp.db") def setUp(self): # Set testing mode to True so endpoints know to look for a custom # database instance to work with - self.app.config['TESTING'] = True + self.app.config["TESTING"] = True # Make sure valid referrers is set as tests expect config = DOIConfigUtil.get_config() - config.set('OTHER', 'api_valid_referrers', 'localhost,0.0.0.0') + config.set("OTHER", "api_valid_referrers", "localhost,0.0.0.0") def tearDown(self): # Remove the temp DB so a new one is created before each test @@ -69,18 +71,20 @@ def list_action_run_patch(self, **kwargs): """ return json.dumps( [ - {"status": DoiStatus.Draft, - "date_added": '2020-10-20T14:04:12.560568-07:00', - "date_updated": '2020-10-20T14:04:12.560568-07:00', - "submitter": "eng-submitter@jpl.nasa.gov", - "title": "InSight Cameras Bundle 1.1", "type": "Dataset", - "subtype": "PDS4 Refereed Data Bundle", "node_id": "eng", - "identifier": "urn:nasa:pds:insight_cameras::1.1", - "doi": '10.17189/28957', - "transaction_key": join( - TestDoisController.test_data_dir, TestDoisController.service_type - ), - "is_latest": 1} + { + "status": DoiStatus.Draft, + "date_added": "2020-10-20T14:04:12.560568-07:00", + "date_updated": "2020-10-20T14:04:12.560568-07:00", + "submitter": "eng-submitter@jpl.nasa.gov", + "title": "InSight Cameras Bundle 1.1", + "type": "Dataset", + "subtype": "PDS4 Refereed Data Bundle", + "node_id": "eng", + "identifier": "urn:nasa:pds:insight_cameras::1.1", + "doi": "10.17189/28957", + "transaction_key": join(TestDoisController.test_data_dir, TestDoisController.service_type), + "is_latest": 1, + } ] ) @@ -90,7 +94,7 @@ def list_action_run_patch_missing(self, **kwargs): Returns a result corresponding to an unsuccessful search. """ - return '[]' + return "[]" def draft_action_run_patch(self, **kwargs): """ @@ -99,10 +103,8 @@ def draft_action_run_patch(self, **kwargs): Returns body of a label corresponding to a successful draft request. """ - draft_record_file = join( - TestDoisController.test_data_dir, TestDoisController.service_type, - 'draft_record') - with open(draft_record_file, 'r') as infile: + draft_record_file = join(TestDoisController.test_data_dir, TestDoisController.service_type, "draft_record") + with open(draft_record_file, "r") as infile: return infile.read() def reserve_action_run_patch(self, **kwargs): @@ -112,10 +114,8 @@ def reserve_action_run_patch(self, **kwargs): Returns body of a label corresponding to a successful reserve (dry-run) request. """ - draft_record_file = join( - TestDoisController.test_data_dir, TestDoisController.service_type, - 'reserve_record') - with open(draft_record_file, 'r') as infile: + draft_record_file = join(TestDoisController.test_data_dir, TestDoisController.service_type, "reserve_record") + with open(draft_record_file, "r") as infile: return infile.read() def release_action_run_patch(self, **kwargs): @@ -125,10 +125,8 @@ def release_action_run_patch(self, **kwargs): Returns body of a label corresponding to a successful release request. """ - draft_record_file = join( - TestDoisController.test_data_dir, TestDoisController.service_type, - 'release_record') - with open(draft_record_file, 'r') as infile: + draft_record_file = join(TestDoisController.test_data_dir, TestDoisController.service_type, "release_record") + with open(draft_record_file, "r") as infile: return infile.read() def release_action_run_w_error_patch(self, **kwargs): @@ -138,10 +136,8 @@ def release_action_run_w_error_patch(self, **kwargs): Returns body of a label corresponding to errors returned from the DOI service provider. """ - draft_record_file = join( - TestDoisController.test_data_dir, TestDoisController.service_type, - 'error_record') - with open(draft_record_file, 'r') as infile: + draft_record_file = join(TestDoisController.test_data_dir, TestDoisController.service_type, "error_record") + with open(draft_record_file, "r") as infile: return infile.read() def transaction_log_patch(self): @@ -157,16 +153,16 @@ def webclient_query_patch(self, query=None, content_type=CONTENT_TYPE_XML): """ # Return dummy xml results containing the statuses we expect # Released - if query['doi'] == '10.17189/28957': - xml_file = 'DOI_Release_20200727_from_register.xml' + if query["doi"] == "10.17189/28957": + xml_file = "DOI_Release_20200727_from_register.xml" # Pending - elif query['doi'] == '10.17189/29348': - xml_file = 'DOI_Release_20200727_from_release.xml' + elif query["doi"] == "10.17189/29348": + xml_file = "DOI_Release_20200727_from_release.xml" # Error else: - xml_file = 'DOI_Release_20200727_from_error.xml' + xml_file = "DOI_Release_20200727_from_error.xml" - with open(join(TestDoisController.input_dir, xml_file), 'r') as infile: + with open(join(TestDoisController.input_dir, xml_file), "r") as infile: xml_contents = infile.read() return xml_contents @@ -175,45 +171,39 @@ def test_get_dois(self): """Test case for get_dois""" # For these tests, use a pre-existing database with some canned # entries to query for - test_db = join(self.test_data_dir, 'test.db') + test_db = join(self.test_data_dir, "test.db") # Start with a empty query to fetch all available records - query_string = [('db_name', test_db)] + query_string = [("db_name", test_db)] # Ensure fetch-all endpoint works both with and without a trailing # slash - endpoints = ['/PDS_APIs/pds_doi_api/0.2/dois', - '/PDS_APIs/pds_doi_api/0.2/dois/'] + endpoints = ["/PDS_APIs/pds_doi_api/0.2/dois", "/PDS_APIs/pds_doi_api/0.2/dois/"] for endpoint in endpoints: - response = self.client.open(endpoint, method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + endpoint, method="GET", query_string=query_string, headers={"Referer": "http://localhost"} ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + records = response.json # Test database should contain 3 records self.assertEqual(len(records), 3) # Now use a query string to ensure we can get specific records back - query_string = [('node', 'eng'), - ('db_name', test_db)] + query_string = [("node", "eng"), ("db_name", test_db)] - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="GET", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + # Should only get one of the records back records = response.json self.assertEqual(len(records), 1) @@ -221,30 +211,31 @@ def test_get_dois(self): # Reformat JSON result into a DoiSummary object so we can check fields summary = DoiSummary.from_dict(records[0]) - self.assertEqual(summary.node, 'eng') - self.assertEqual(summary.title, 'InSight Cameras Bundle 1.1') - self.assertEqual(summary.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(summary.identifier, 'urn:nasa:pds:insight_cameras::1.1') + self.assertEqual(summary.node, "eng") + self.assertEqual(summary.title, "InSight Cameras Bundle 1.1") + self.assertEqual(summary.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(summary.identifier, "urn:nasa:pds:insight_cameras::1.1") self.assertEqual(summary.status, DoiStatus.Draft) # Test filtering by start/end date # Note: this test was originally developed on PDT, so its important # to include the correct time zone offset as part of the query - query_string = [('start_date', '2020-10-20T21:04:13.000000+08:00'), - ('end_date', '2020-10-20T21:04:14.000000+08:00'), - ('db_name', test_db)] + query_string = [ + ("start_date", "2020-10-20T21:04:13.000000+08:00"), + ("end_date", "2020-10-20T21:04:14.000000+08:00"), + ("db_name", test_db), + ] - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="GET", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + # Should only get one of the records back records = response.json self.assertEqual(len(records), 1) @@ -252,27 +243,24 @@ def test_get_dois(self): # Reformat JSON result into a DoiSummary object so we can check fields summary = DoiSummary.from_dict(records[0]) - self.assertEqual(summary.node, 'img') - self.assertEqual(summary.title, 'InSight Cameras Bundle 1.0') - self.assertEqual(summary.submitter, 'img-submitter@jpl.nasa.gov') - self.assertEqual(summary.identifier, 'urn:nasa:pds:insight_cameras::1.0') + self.assertEqual(summary.node, "img") + self.assertEqual(summary.title, "InSight Cameras Bundle 1.0") + self.assertEqual(summary.submitter, "img-submitter@jpl.nasa.gov") + self.assertEqual(summary.identifier, "urn:nasa:pds:insight_cameras::1.0") self.assertEqual(summary.status, DoiStatus.Reserved_not_submitted) # Test fetching of a record that only has an LID (no VID) associated to it - query_string = [('node', 'img'), - ('ids', 'urn:nasa:pds:lab_shocked_feldspars'), - ('db_name', test_db)] - - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + query_string = [("node", "img"), ("ids", "urn:nasa:pds:lab_shocked_feldspars"), ("db_name", test_db)] + + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="GET", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + # Should only get one of the records back records = response.json self.assertEqual(len(records), 1) @@ -280,269 +268,259 @@ def test_get_dois(self): # Reformat JSON result into a DoiSummary object so we can check fields summary = DoiSummary.from_dict(records[0]) - self.assertEqual(summary.node, 'img') - self.assertEqual(summary.title, 'Laboratory Shocked Feldspars Bundle') - self.assertEqual(summary.submitter, 'img-submitter@jpl.nasa.gov') - self.assertEqual(summary.identifier, 'urn:nasa:pds:lab_shocked_feldspars') + self.assertEqual(summary.node, "img") + self.assertEqual(summary.title, "Laboratory Shocked Feldspars Bundle") + self.assertEqual(summary.submitter, "img-submitter@jpl.nasa.gov") + self.assertEqual(summary.identifier, "urn:nasa:pds:lab_shocked_feldspars") self.assertEqual(summary.status, DoiStatus.Reserved_not_submitted) # Now try filtering by workflow status - query_string = [('status', DoiStatus.Reserved_not_submitted.value), - ('db_name', test_db)] - - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) + query_string = [("status", DoiStatus.Reserved_not_submitted.value), ("db_name", test_db)] - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="GET", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + # Should only get two of the records back records = response.json self.assertEqual(len(records), 2) # Finally, test with a malformed start/end date and ensure we # get "invalid argument" code back - query_string = [('start_date', '2020-10-20 14:04:13.000000'), - ('end_date', '10-20-2020 14:04'), - ('db_name', test_db)] - - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert400( - response, - 'Response body is : ' + response.data.decode('utf-8') + query_string = [ + ("start_date", "2020-10-20 14:04:13.000000"), + ("end_date", "10-20-2020 14:04"), + ("db_name", test_db), + ] + + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="GET", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionDraft, - 'run', draft_action_run_patch) + self.assert400(response, "Response body is : " + response.data.decode("utf-8")) + + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionDraft, "run", draft_action_run_patch) def test_post_dois_draft_w_url(self): """Test a draft POST with url input""" # We can use a file system path since were working with a local server - input_bundle = join(self.test_data_dir, 'bundle_in.xml') + input_bundle = join(self.test_data_dir, "bundle_in.xml") # Start by submitting a draft request - query_string = [('action', 'draft'), - ('submitter', 'eng-submitter@jpl.nasa.gov'), - ('node', 'eng'), - ('url', input_bundle), - ('db_name', self.temp_db)] - - draft_response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='POST', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - draft_response, - 'Response body is : ' + draft_response.data.decode('utf-8') + query_string = [ + ("action", "draft"), + ("submitter", "eng-submitter@jpl.nasa.gov"), + ("node", "eng"), + ("url", input_bundle), + ("db_name", self.temp_db), + ] + + draft_response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="POST", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(draft_response, "Response body is : " + draft_response.data.decode("utf-8")) + # Recreate a DoiRecord from the response JSON and examine the # fields draft_record = DoiRecord.from_dict(draft_response.json[0]) - self.assertEqual(draft_record.node, 'eng') - self.assertEqual(draft_record.title, 'InSight Cameras Bundle 1.1') - self.assertEqual(draft_record.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(draft_record.identifier, 'urn:nasa:pds:insight_cameras::1.1') - self.assertEqual(draft_record.doi, '10.17189/28957') - self.assertEqual(draft_record.creation_date, - datetime.fromisoformat('2020-10-20T14:04:12.560568-07:00')) - self.assertEqual(draft_record.update_date, - datetime.fromisoformat('2020-10-20T14:04:12.560568-07:00')) + self.assertEqual(draft_record.node, "eng") + self.assertEqual(draft_record.title, "InSight Cameras Bundle 1.1") + self.assertEqual(draft_record.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(draft_record.identifier, "urn:nasa:pds:insight_cameras::1.1") + self.assertEqual(draft_record.doi, "10.17189/28957") + self.assertEqual(draft_record.creation_date, datetime.fromisoformat("2020-10-20T14:04:12.560568-07:00")) + self.assertEqual(draft_record.update_date, datetime.fromisoformat("2020-10-20T14:04:12.560568-07:00")) # Note we get Pending back from the parsed label, however # the object sent to transaction database has 'Draft' status self.assertEqual(draft_record.status, DoiStatus.Pending) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionDraft, - 'run', draft_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionDraft, "run", draft_action_run_patch) def test_post_dois_draft_w_payload(self): """Test a draft POST with requestBody input""" - input_bundle = join(self.test_data_dir, 'bundle_in.xml') + input_bundle = join(self.test_data_dir, "bundle_in.xml") - with open(input_bundle, 'rb') as infile: + with open(input_bundle, "rb") as infile: body = infile.read() - query_string = [('action', 'draft'), - ('submitter', 'eng-submitter@jpl.nasa.gov'), - ('node', 'eng'), - ('db_name', self.temp_db)] - - draft_response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='POST', - data=body, - content_type='application/xml', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - draft_response, - 'Response body is : ' + draft_response.data.decode('utf-8') + query_string = [ + ("action", "draft"), + ("submitter", "eng-submitter@jpl.nasa.gov"), + ("node", "eng"), + ("db_name", self.temp_db), + ] + + draft_response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="POST", + data=body, + content_type="application/xml", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(draft_response, "Response body is : " + draft_response.data.decode("utf-8")) + # Recreate a DoiRecord from the response JSON and examine the # fields draft_record = DoiRecord.from_dict(draft_response.json[0]) - self.assertEqual(draft_record.node, 'eng') - self.assertEqual(draft_record.title, 'InSight Cameras Bundle 1.1') - self.assertEqual(draft_record.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(draft_record.identifier, 'urn:nasa:pds:insight_cameras::1.1') - self.assertEqual(draft_record.doi, '10.17189/28957') - self.assertEqual(draft_record.creation_date, - datetime.fromisoformat('2020-10-20T14:04:12.560568-07:00')) - self.assertEqual(draft_record.update_date, - datetime.fromisoformat('2020-10-20T14:04:12.560568-07:00')) + self.assertEqual(draft_record.node, "eng") + self.assertEqual(draft_record.title, "InSight Cameras Bundle 1.1") + self.assertEqual(draft_record.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(draft_record.identifier, "urn:nasa:pds:insight_cameras::1.1") + self.assertEqual(draft_record.doi, "10.17189/28957") + self.assertEqual(draft_record.creation_date, datetime.fromisoformat("2020-10-20T14:04:12.560568-07:00")) + self.assertEqual(draft_record.update_date, datetime.fromisoformat("2020-10-20T14:04:12.560568-07:00")) # Note we get Pending back from the parsed label, however # the object sent to transaction database has 'Draft' status self.assertEqual(draft_record.status, DoiStatus.Pending) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionReserve, - 'run', reserve_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionReserve, "run", reserve_action_run_patch) def test_post_dois_reserve(self): """Test dry-run reserve POST""" # Submit a new bundle in reserve (not submitted) status body = LabelsPayload( - [LabelPayload(status=DoiStatus.Reserved, - title='Laboratory Shocked Feldspars Bundle', - publication_date=datetime.now(), - product_type_specific='PDS4 Bundle', - author_last_name='Johnson', - author_first_name='J. R.', - related_resource='urn:nasa:pds:lab_shocked_feldspars')] + [ + LabelPayload( + status=DoiStatus.Reserved, + title="Laboratory Shocked Feldspars Bundle", + publication_date=datetime.now(), + product_type_specific="PDS4 Bundle", + author_last_name="Johnson", + author_first_name="J. R.", + related_resource="urn:nasa:pds:lab_shocked_feldspars", + ) + ] ) - query_string = [('action', 'reserve'), - ('submitter', 'img-submitter@jpl.nasa.gov'), - ('node', 'img'), - ('db_name', self.temp_db)] - - reserve_response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='POST', - data=JSONEncoder().encode(body), - content_type='application/json', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert200( - reserve_response, - 'Response body is : ' + reserve_response.data.decode('utf-8') + query_string = [ + ("action", "reserve"), + ("submitter", "img-submitter@jpl.nasa.gov"), + ("node", "img"), + ("db_name", self.temp_db), + ] + + reserve_response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="POST", + data=JSONEncoder().encode(body), + content_type="application/json", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(reserve_response, "Response body is : " + reserve_response.data.decode("utf-8")) + # Recreate a DoiRecord from the response JSON and examine the # fields reserve_record = DoiRecord.from_dict(reserve_response.json[0]) - self.assertEqual(reserve_record.node, 'img') - self.assertEqual(reserve_record.title, 'InSight Cameras Bundle') - self.assertEqual(reserve_record.submitter, 'img-submitter@jpl.nasa.gov') - self.assertEqual(reserve_record.identifier, 'urn:nasa:pds:insight_cameras::2.0') + self.assertEqual(reserve_record.node, "img") + self.assertEqual(reserve_record.title, "InSight Cameras Bundle") + self.assertEqual(reserve_record.submitter, "img-submitter@jpl.nasa.gov") + self.assertEqual(reserve_record.identifier, "urn:nasa:pds:insight_cameras::2.0") self.assertEqual(reserve_record.status, DoiStatus.Reserved_not_submitted) def test_post_dois_invalid_requests(self): """Test invalid POST requests""" # Test with an unknown action, should get Invalid Argument - query_string = [('action', 'unknown'), - ('submitter', 'img-submitter@jpl.nasa.gov'), - ('node', 'img'), - ('db_name', self.temp_db)] - - error_response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='POST', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert400( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') + query_string = [ + ("action", "unknown"), + ("submitter", "img-submitter@jpl.nasa.gov"), + ("node", "img"), + ("db_name", self.temp_db), + ] + + error_response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="POST", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert400(error_response, "Response body is : " + error_response.data.decode("utf-8")) + # Test draft action with no url or requestBody input - query_string = [('action', 'draft'), - ('submitter', 'img-submitter@jpl.nasa.gov'), - ('node', 'img'), - ('db_name', self.temp_db)] - - error_response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='POST', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert400( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') + query_string = [ + ("action", "draft"), + ("submitter", "img-submitter@jpl.nasa.gov"), + ("node", "img"), + ("db_name", self.temp_db), + ] + + error_response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="POST", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert400(error_response, "Response body is : " + error_response.data.decode("utf-8")) + # Test reserve action with a url instead of a requestBody - input_bundle = join(self.test_data_dir, 'bundle_in.xml') - - query_string = [('action', 'reserve'), - ('submitter', 'eng-submitter@jpl.nasa.gov'), - ('node', 'eng'), - ('url', input_bundle), - ('db_name', self.temp_db)] - - error_response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='POST', - query_string=query_string, - headers={'Referer': 'http://localhost'}) - - self.assert400( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') + input_bundle = join(self.test_data_dir, "bundle_in.xml") + + query_string = [ + ("action", "reserve"), + ("submitter", "eng-submitter@jpl.nasa.gov"), + ("node", "eng"), + ("url", input_bundle), + ("db_name", self.temp_db), + ] + + error_response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", + method="POST", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) + self.assert400(error_response, "Response body is : " + error_response.data.decode("utf-8")) + + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) def test_post_submit(self): """Test the submit endpoint""" - query_string = [('force', False), - ('db_name', self.temp_db), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [ + ("force", False), + ("db_name", self.temp_db), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ] release_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/doi/submit', - method='POST', + "/PDS_APIs/pds_doi_api/0.2/doi/submit", + method="POST", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert200( - release_response, - 'Response body is : ' + release_response.data.decode('utf-8') - ) + self.assert200(release_response, "Response body is : " + release_response.data.decode("utf-8")) # Recreate a DoiRecord from the response JSON and examine the # fields submit_record = DoiRecord.from_dict(release_response.json[0]) - self.assertEqual(submit_record.node, 'eng') - self.assertEqual(submit_record.title, 'InSight Cameras Bundle 1.1') - self.assertEqual(submit_record.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(submit_record.identifier, 'urn:nasa:pds:insight_cameras::1.1') + self.assertEqual(submit_record.node, "eng") + self.assertEqual(submit_record.title, "InSight Cameras Bundle 1.1") + self.assertEqual(submit_record.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(submit_record.identifier, "urn:nasa:pds:insight_cameras::1.1") self.assertEqual(submit_record.status, DoiStatus.Review) - self.assertEqual(submit_record.doi, '10.17189/28957') + self.assertEqual(submit_record.doi, "10.17189/28957") def test_disabled_release_endpoint(self): """ @@ -552,135 +530,122 @@ def test_disabled_release_endpoint(self): endpoint is ever re-enabled, along with the @unittest.skip decorators for the corresponding unit tests. """ - query_string = [('force', False), - ('db_name', self.temp_db), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [ + ("force", False), + ("db_name", self.temp_db), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ] release_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/dois/release', - method='POST', + "/PDS_APIs/pds_doi_api/0.2/dois/release", + method="POST", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert404( - release_response, - 'Response body is : ' + release_response.data.decode('utf-8') - ) + self.assert404(release_response, "Response body is : " + release_response.data.decode("utf-8")) - @unittest.skip('dois/release endpoint is disabled') - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionRelease, - 'run', release_action_run_patch) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) + @unittest.skip("dois/release endpoint is disabled") + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionRelease, "run", release_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) def test_post_release(self): """Test the release endpoint""" - query_string = [('force', False), - ('db_name', self.temp_db), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [ + ("force", False), + ("db_name", self.temp_db), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ] release_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/dois/release', - method='POST', + "/PDS_APIs/pds_doi_api/0.2/dois/release", + method="POST", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert200( - release_response, - 'Response body is : ' + release_response.data.decode('utf-8') - ) + self.assert200(release_response, "Response body is : " + release_response.data.decode("utf-8")) # Recreate a DoiRecord from the response JSON and examine the # fields release_record = DoiRecord.from_dict(release_response.json[0]) - self.assertEqual(release_record.node, 'eng') - self.assertEqual(release_record.title, 'InSight Cameras Bundle 1.1') - self.assertEqual(release_record.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(release_record.identifier, 'urn:nasa:pds:insight_cameras::1.1') + self.assertEqual(release_record.node, "eng") + self.assertEqual(release_record.title, "InSight Cameras Bundle 1.1") + self.assertEqual(release_record.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(release_record.identifier, "urn:nasa:pds:insight_cameras::1.1") self.assertEqual(release_record.status, DoiStatus.Pending) - self.assertEqual(release_record.doi, '10.17189/21734') + self.assertEqual(release_record.doi, "10.17189/21734") # Record field should match what we provided via patch method self.assertEqual(release_record.record, self.release_action_run_patch()) - @unittest.skip('dois/release endpoint is disabled') + @unittest.skip("dois/release endpoint is disabled") @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionRelease, - 'run', release_action_run_w_error_patch) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) + pds_doi_service.api.controllers.dois_controller.DOICoreActionRelease, "run", release_action_run_w_error_patch + ) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) def test_post_release_w_errors(self): """ Test the release endpoint where errors are received back from the release action. """ - query_string = [('force', False), - ('db_name', self.temp_db), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [ + ("force", False), + ("db_name", self.temp_db), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ] error_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/dois/release', - method='POST', + "/PDS_APIs/pds_doi_api/0.2/dois/release", + method="POST", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert400( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') - ) + self.assert400(error_response, "Response body is : " + error_response.data.decode("utf-8")) # Check the error response and make sure it contains all the errors # provided from the original XML - errors = error_response.json['errors'] + errors = error_response.json["errors"] - self.assertEqual(errors[0]['name'], 'WarningDOIException') - self.assertIn('Title is required', errors[0]['message']) - self.assertIn('A publication date is required', errors[0]['message']) - self.assertIn('A site URL is required', errors[0]['message']) - self.assertIn('A product type is required', errors[0]['message']) - self.assertIn('A specific product type is required for non-dataset types', - errors[0]['message']) + self.assertEqual(errors[0]["name"], "WarningDOIException") + self.assertIn("Title is required", errors[0]["message"]) + self.assertIn("A publication date is required", errors[0]["message"]) + self.assertIn("A site URL is required", errors[0]["message"]) + self.assertIn("A product type is required", errors[0]["message"]) + self.assertIn("A specific product type is required for non-dataset types", errors[0]["message"]) - @unittest.skip('dois/release endpoint is disabled') + @unittest.skip("dois/release endpoint is disabled") @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch_missing) + pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch_missing + ) def test_post_release_missing_lid(self): """ Test the release endpoint where no existing entry for the requested LID exists. """ - query_string = [('force', False), - ('db_name', self.temp_db), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [ + ("force", False), + ("db_name", self.temp_db), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ] error_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/dois/release', - method='POST', + "/PDS_APIs/pds_doi_api/0.2/dois/release", + method="POST", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert404( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') - ) + self.assert404(error_response, "Response body is : " + error_response.data.decode("utf-8")) # Check the error response and make sure it contains the expected # error message - errors = error_response.json['errors'] + errors = error_response.json["errors"] - self.assertEqual(errors[0]['name'], 'UnknownLIDVIDException') + self.assertEqual(errors[0]["name"], "UnknownLIDVIDException") self.assertIn( - 'No record(s) could be found for LIDVID ' - 'urn:nasa:pds:insight_cameras::1.1', - errors[0]['message'] + "No record(s) could be found for LIDVID " "urn:nasa:pds:insight_cameras::1.1", errors[0]["message"] ) def list_action_run_patch_no_transaction_history(self, **kwargs): @@ -690,76 +655,66 @@ def list_action_run_patch_no_transaction_history(self, **kwargs): Returns a result corresponding to an entry where the listed transaction_key location no longer exists. """ - return json.dumps( - [ - {"transaction_key": '/dev/null', "is_latest": 1} - ] - ) + return json.dumps([{"transaction_key": "/dev/null", "is_latest": 1}]) - @unittest.skip('dois/release endpoint is disabled') + @unittest.skip("dois/release endpoint is disabled") @patch.object( pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch_no_transaction_history) + "run", + list_action_run_patch_no_transaction_history, + ) def test_post_release_missing_transaction_history(self): """ Test the release endpoint where the requested LID returns an entry with a missing transaction_key location. """ - query_string = [('force', False), - ('db_name', self.temp_db), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [ + ("force", False), + ("db_name", self.temp_db), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ] error_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/dois/release', - method='POST', + "/PDS_APIs/pds_doi_api/0.2/dois/release", + method="POST", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert500( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') - ) + self.assert500(error_response, "Response body is : " + error_response.data.decode("utf-8")) # Check the error response and make sure it contains the expected # error message - errors = error_response.json['errors'] + errors = error_response.json["errors"] - self.assertEqual(errors[0]['name'], 'NoTransactionHistoryForLIDVIDException') + self.assertEqual(errors[0]["name"], "NoTransactionHistoryForLIDVIDException") self.assertIn( - 'Could not find a DOI label associated with identifier ' - 'urn:nasa:pds:insight_cameras::1.1', - errors[0]['message'] + "Could not find a DOI label associated with identifier " "urn:nasa:pds:insight_cameras::1.1", + errors[0]["message"], ) - @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch) + @patch.object(pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch) def test_get_doi_from_id(self): """Test case for get_doi_from_id""" - query_string = [('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [("identifier", "urn:nasa:pds:insight_cameras::1.1")] response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/doi' - .format(lidvid='urn:nasa:pds:insight_cameras::1.1'), - method='GET', + "/PDS_APIs/pds_doi_api/0.2/doi".format(lidvid="urn:nasa:pds:insight_cameras::1.1"), + method="GET", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') - ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) # Recreate a DoiRecord from the response JSON and examine the # fields record = DoiRecord.from_dict(response.json) - self.assertEqual(record.node, 'eng') - self.assertEqual(record.title, 'InSight Cameras Bundle 1.1') - self.assertEqual(record.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(record.identifier, 'urn:nasa:pds:insight_cameras::1.1') + self.assertEqual(record.node, "eng") + self.assertEqual(record.title, "InSight Cameras Bundle 1.1") + self.assertEqual(record.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(record.identifier, "urn:nasa:pds:insight_cameras::1.1") self.assertEqual(record.status, DoiStatus.Pending) # Make sure we only got one record back @@ -768,26 +723,23 @@ def test_get_doi_from_id(self): self.assertEqual(len(dois), 1) # Test again with an LID only, should get the same result back - query_string = [('identifier', 'urn:nasa:pds:insight_cameras')] + query_string = [("identifier", "urn:nasa:pds:insight_cameras")] response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/doi', - method='GET', + "/PDS_APIs/pds_doi_api/0.2/doi", + method="GET", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') - ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) record = DoiRecord.from_dict(response.json) - self.assertEqual(record.node, 'eng') - self.assertEqual(record.title, 'InSight Cameras Bundle') - self.assertEqual(record.submitter, 'eng-submitter@jpl.nasa.gov') - self.assertEqual(record.identifier, 'urn:nasa:pds:insight_cameras') + self.assertEqual(record.node, "eng") + self.assertEqual(record.title, "InSight Cameras Bundle") + self.assertEqual(record.submitter, "eng-submitter@jpl.nasa.gov") + self.assertEqual(record.identifier, "urn:nasa:pds:insight_cameras") self.assertEqual(record.status, DoiStatus.Pending) # Make sure we only got one record back @@ -796,138 +748,131 @@ def test_get_doi_from_id(self): self.assertEqual(len(dois), 1) @patch.object( - pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch_missing) + pds_doi_service.api.controllers.dois_controller.DOICoreActionList, "run", list_action_run_patch_missing + ) def test_get_doi_missing_id(self): """Test get_doi_from_id where requested LIDVID is not found""" - query_string = [('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [("identifier", "urn:nasa:pds:insight_cameras::1.1")] error_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/doi', - method='GET', + "/PDS_APIs/pds_doi_api/0.2/doi", + method="GET", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert404( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') - ) + self.assert404(error_response, "Response body is : " + error_response.data.decode("utf-8")) # Check the error response and make sure it contains the expected # error message - errors = error_response.json['errors'] + errors = error_response.json["errors"] - self.assertEqual(errors[0]['name'], 'UnknownIdentifierException') + self.assertEqual(errors[0]["name"], "UnknownIdentifierException") self.assertIn( - 'No record(s) could be found for identifier ' - 'urn:nasa:pds:insight_cameras::1.1', - errors[0]['message'] + "No record(s) could be found for identifier " "urn:nasa:pds:insight_cameras::1.1", errors[0]["message"] ) @patch.object( pds_doi_service.api.controllers.dois_controller.DOICoreActionList, - 'run', list_action_run_patch_no_transaction_history) + "run", + list_action_run_patch_no_transaction_history, + ) def test_get_doi_missing_transaction_history(self): """ Test get_doi_from_id where transaction history for LIDVID cannot be found """ - query_string = [('identifier', 'urn:nasa:pds:insight_cameras::1.1')] + query_string = [("identifier", "urn:nasa:pds:insight_cameras::1.1")] error_response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/doi', - method='GET', + "/PDS_APIs/pds_doi_api/0.2/doi", + method="GET", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) - self.assert500( - error_response, - 'Response body is : ' + error_response.data.decode('utf-8') - ) + self.assert500(error_response, "Response body is : " + error_response.data.decode("utf-8")) # Check the error response and make sure it contains the expected # error message - errors = error_response.json['errors'] + errors = error_response.json["errors"] - self.assertEqual(errors[0]['name'], 'NoTransactionHistoryForIdentifierException') + self.assertEqual(errors[0]["name"], "NoTransactionHistoryForIdentifierException") self.assertIn( - 'Could not find a DOI label associated with identifier ' - 'urn:nasa:pds:insight_cameras::1.1', - errors[0]['message'] + "Could not find a DOI label associated with identifier " "urn:nasa:pds:insight_cameras::1.1", + errors[0]["message"], ) def test_put_doi_from_id(self): """Test case for put_doi_from_id""" - query_string = [('submitter', 'img-submitter@jpl.nasa.gov'), - ('node', 'img'), - ('identifier', 'urn:nasa:pds:insight_cameras::1.1'), - ('url', 'http://fake.url.net')] + query_string = [ + ("submitter", "img-submitter@jpl.nasa.gov"), + ("node", "img"), + ("identifier", "urn:nasa:pds:insight_cameras::1.1"), + ("url", "http://fake.url.net"), + ] response = self.client.open( - '/PDS_APIs/pds_doi_api/0.2/doi', - method='PUT', + "/PDS_APIs/pds_doi_api/0.2/doi", + method="PUT", query_string=query_string, - headers={'Referer': 'http://localhost'} + headers={"Referer": "http://localhost"}, ) # Should return a Not Implemented code self.assertEqual(response.status_code, 501) - errors = response.json['errors'] - self.assertEqual(errors[0]['name'], 'NotImplementedError') - self.assertIn( - 'Please use the POST /doi endpoint for record update', - errors[0]['message'] - ) + errors = response.json["errors"] + self.assertEqual(errors[0]["name"], "NotImplementedError") + self.assertIn("Please use the POST /doi endpoint for record update", errors[0]["message"]) - @unittest.skipIf(DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE, - "DataCite does not assign a pending state to release requests") + @unittest.skipIf( + DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE, + "DataCite does not assign a pending state to release requests", + ) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'query_doi', webclient_query_patch) - @patch.object( - pds_doi_service.core.outputs.transaction.Transaction, - 'log', transaction_log_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "query_doi", webclient_query_patch + ) + @patch.object(pds_doi_service.core.outputs.transaction.Transaction, "log", transaction_log_patch) def test_get_check_dois(self): """Test case for get_check_dois""" # TODO need datacite version - test_db = join(self.test_data_dir, 'pending_dois.db') - - query_string = [('submitter', 'doi-checker@jpl.nasa.gov'), - ('email', False), - ('attachment', False), - ('db_name', test_db)] + test_db = join(self.test_data_dir, "pending_dois.db") - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois/check', - method='GET', - query_string=query_string, - headers={'Referer': 'http://localhost'}) + query_string = [ + ("submitter", "doi-checker@jpl.nasa.gov"), + ("email", False), + ("attachment", False), + ("db_name", test_db), + ] - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois/check", + method="GET", + query_string=query_string, + headers={"Referer": "http://localhost"}, ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + records = response.json self.assertEqual(len(records), 3) # Check each record and ensure each DOI returned with the expected # status for record in records: - self.assertEqual(record['submitter'], 'doi-checker@jpl.nasa.gov') + self.assertEqual(record["submitter"], "doi-checker@jpl.nasa.gov") - if record['doi'] == '10.17189/28957': - self.assertEqual(record['status'], DoiStatus.Registered) + if record["doi"] == "10.17189/28957": + self.assertEqual(record["status"], DoiStatus.Registered) - if record['doi'] == '10.17189/29348': - self.assertEqual(record['status'], DoiStatus.Pending) + if record["doi"] == "10.17189/29348": + self.assertEqual(record["status"], DoiStatus.Pending) - if record['doi'] == '10.17189/29527': - self.assertEqual(record['status'], DoiStatus.Error) + if record["doi"] == "10.17189/29527": + self.assertEqual(record["status"], DoiStatus.Error) # Make sure we got a message back with the error - self.assertIsNotNone(record['message']) + self.assertIsNotNone(record["message"]) def test_filter_by_referrers(self): """Test filtering of requests based on the referer header value""" @@ -935,36 +880,27 @@ def test_filter_by_referrers(self): # By default, the INI config should specify localhost and 0.0.0.0 as # valid hostnames, so attempting with any other referrer should # return a 403 "forbidden" error - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - headers={'Referer': 'http://www.zombo.com'}) - - self.assert403( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", method="GET", headers={"Referer": "http://www.zombo.com"} ) + self.assert403(response, "Response body is : " + response.data.decode("utf-8")) + # Requests with no referrer provided should also fail with a 401 # "unauthorized" error - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET') + response = self.client.open("/PDS_APIs/pds_doi_api/0.2/dois", method="GET") - self.assert401( - response, - 'Response body is : ' + response.data.decode('utf-8') - ) + self.assert401(response, "Response body is : " + response.data.decode("utf-8")) # Providing a valid referrer should make everything work again - response = self.client.open('/PDS_APIs/pds_doi_api/0.2/dois', - method='GET', - headers={'Referer': 'http://0.0.0.0'}) - - self.assert200( - response, - 'Response body is : ' + response.data.decode('utf-8') + response = self.client.open( + "/PDS_APIs/pds_doi_api/0.2/dois", method="GET", headers={"Referer": "http://0.0.0.0"} ) + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) + -if __name__ == '__main__': +if __name__ == "__main__": import unittest + unittest.main() diff --git a/src/pds_doi_service/api/util.py b/src/pds_doi_service/api/util.py index c39b9d96..fe06dc80 100755 --- a/src/pds_doi_service/api/util.py +++ b/src/pds_doi_service/api/util.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ======= util.py @@ -13,12 +12,11 @@ Utility functions for the PDS DOI API service. This module is adapted from the SwaggerHub auto-generated util.py. """ - import datetime - +import typing from collections import Iterable + import six -import typing def format_exceptions(exceptions): @@ -54,12 +52,7 @@ def format_exceptions(exceptions): if not isinstance(exceptions, Iterable): exceptions = [exceptions] - return { - 'errors': [ - {'name': type(exception).__name__, 'message': str(exception)} - for exception in exceptions - ] - } + return {"errors": [{"name": type(exception).__name__, "message": str(exception)} for exception in exceptions]} def _deserialize(data, klass): @@ -126,6 +119,7 @@ def deserialize_date(string): """ try: from dateutil.parser import parse + return parse(string).date() except ImportError: return string @@ -143,6 +137,7 @@ def deserialize_datetime(string): """ try: from dateutil.parser import parse + return parse(string) except ImportError: return string @@ -162,9 +157,7 @@ def deserialize_model(data, klass): return data for attr, attr_type in six.iteritems(instance.swagger_types): - if data is not None \ - and instance.attribute_map[attr] in data \ - and isinstance(data, (list, dict)): + if data is not None and instance.attribute_map[attr] in data and isinstance(data, (list, dict)): value = data[instance.attribute_map[attr]] setattr(instance, attr, _deserialize(value, attr_type)) @@ -181,8 +174,7 @@ def _deserialize_list(data, boxed_type): :return: deserialized list. :rtype: list """ - return [_deserialize(sub_data, boxed_type) - for sub_data in data] + return [_deserialize(sub_data, boxed_type) for sub_data in data] def _deserialize_dict(data, boxed_type): @@ -195,5 +187,4 @@ def _deserialize_dict(data, boxed_type): :return: deserialized dict. :rtype: dict """ - return {k: _deserialize(v, boxed_type) - for k, v in six.iteritems(data)} + return {k: _deserialize(v, boxed_type) for k, v in six.iteritems(data)} diff --git a/src/pds_doi_service/core/actions/__init__.py b/src/pds_doi_service/core/actions/__init__.py index 5fa7e0c7..6b3a3ed9 100644 --- a/src/pds_doi_service/core/actions/__init__.py +++ b/src/pds_doi_service/core/actions/__init__.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ======= actions @@ -13,8 +12,8 @@ This package contains the implementations for the user-facing action classes used to interact with the PDS DOI service. """ - -from pds_doi_service.core.actions.action import DOICoreAction, create_parser +from pds_doi_service.core.actions.action import create_parser +from pds_doi_service.core.actions.action import DOICoreAction from pds_doi_service.core.actions.check import DOICoreActionCheck from pds_doi_service.core.actions.draft import DOICoreActionDraft from pds_doi_service.core.actions.list import DOICoreActionList diff --git a/src/pds_doi_service/core/actions/action.py b/src/pds_doi_service/core/actions/action.py index 74c331e2..1390d458 100644 --- a/src/pds_doi_service/core/actions/action.py +++ b/src/pds_doi_service/core/actions/action.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ========= action.py @@ -12,7 +11,6 @@ Contains the parent class definition for actions of the Core PDS DOI Service. """ - import argparse from pds_doi_service.core.outputs.transaction_builder import TransactionBuilder @@ -24,15 +22,15 @@ def create_parser(): """Non-object function to be used by sphinx-argparse for documentation""" - logger.info('parser for sphinx-argparse') + logger.info("parser for sphinx-argparse") return DOICoreAction.create_cmd_parser() class DOICoreAction: m_doi_config_util = DOIConfigUtil() - _name = 'unknown' - _description = 'no description' + _name = "unknown" + _description = "no description" _order = 9999999 # used to sort actions in documentation _run_arguments = () @@ -55,18 +53,17 @@ def create_cmd_parser(): """ parser = argparse.ArgumentParser( - description='PDS core command for DOI management. ' - 'The available subcommands are:\n', - formatter_class=argparse.RawTextHelpFormatter + description="PDS core command for DOI management. " "The available subcommands are:\n", + formatter_class=argparse.RawTextHelpFormatter, ) - subparsers = parser.add_subparsers(dest='subcommand') + subparsers = parser.add_subparsers(dest="subcommand") # create subparsers action_classes = sorted(DOICoreAction.__subclasses__(), key=lambda c: c._order) for cls in action_classes: - parser.description += f'{cls._name} ({cls._description}),\n' + parser.description += f"{cls._name} ({cls._description}),\n" add_to_subparser_method = getattr(cls, "add_to_subparser", None) if callable(add_to_subparser_method): @@ -92,8 +89,7 @@ def add_to_subparser(cls, subparsers): """ return NotImplementedError( - f'Subclasses of {cls.__class__.__name__} must provide an ' - f'implementation for add_to_subparser()' + f"Subclasses of {cls.__class__.__name__} must provide an " f"implementation for add_to_subparser()" ) def parse_arguments(self, kwargs): @@ -110,7 +106,7 @@ def parse_arguments(self, kwargs): """ for kwarg in self._run_arguments: if kwarg in kwargs: - setattr(self, f'_{kwarg}', kwargs[kwarg]) + setattr(self, f"_{kwarg}", kwargs[kwarg]) logger.debug(f"{kwarg} = {getattr(self, f'_{kwarg}')}") @@ -138,6 +134,5 @@ def run(self, **kwargs): """ return NotImplementedError( - f'Subclasses of {self.__class__.__name__} must provide an ' - f'implementation for run()' + f"Subclasses of {self.__class__.__name__} must provide an " f"implementation for run()" ) diff --git a/src/pds_doi_service/core/actions/check.py b/src/pds_doi_service/core/actions/check.py index 4251e9b1..59351323 100644 --- a/src/pds_doi_service/core/actions/check.py +++ b/src/pds_doi_service/core/actions/check.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ======== check.py @@ -12,18 +11,16 @@ Contains the definition for the Check action of the Core PDS DOI Service. """ - import json from copy import deepcopy -from datetime import date, datetime +from datetime import date +from datetime import datetime from email.message import EmailMessage -from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart from os.path import exists -from pkg_resources import resource_filename import pystache - from pds_doi_service.core.actions import DOICoreAction from pds_doi_service.core.actions.list import DOICoreActionList from pds_doi_service.core.entities.doi import DoiStatus @@ -31,17 +28,20 @@ from pds_doi_service.core.outputs.service import DOIServiceFactory from pds_doi_service.core.util.emailer import Emailer from pds_doi_service.core.util.general_util import get_logger +from pkg_resources import resource_filename logger = get_logger(__name__) class DOICoreActionCheck(DOICoreAction): - _name = 'check' - _description = ('Check pending DOI statuses from the service provider and ' - 'update the local database. May be run regularly, for ' - 'example in a crontab.') + _name = "check" + _description = ( + "Check pending DOI statuses from the service provider and " + "update the local database. May be run regularly, for " + "example in a crontab." + ) _order = 30 - _run_arguments = ('submitter', 'email', 'attachment') + _run_arguments = ("submitter", "email", "attachment") def __init__(self, db_name=None): super().__init__(db_name=db_name) @@ -51,51 +51,50 @@ def __init__(self, db_name=None): self._web_client = DOIServiceFactory.get_web_client_service() self._web_parser = DOIServiceFactory.get_web_parser_service() - self._submitter = self._config.get('OTHER', 'emailer_sender') + self._submitter = self._config.get("OTHER", "emailer_sender") self._email = True self._attachment = True - self.email_header_template_file = resource_filename( - __name__, 'email_template_header.mustache' - ) + self.email_header_template_file = resource_filename(__name__, "email_template_header.mustache") - self.email_body_template_file = resource_filename( - __name__, 'email_template_body.txt' - ) + self.email_body_template_file = resource_filename(__name__, "email_template_body.txt") # Make sure templates are where we expect them to be - if (not exists(self.email_header_template_file) - or not exists(self.email_body_template_file)): + if not exists(self.email_header_template_file) or not exists(self.email_body_template_file): raise RuntimeError( - f'Could not find one or more email templates needed by this action\n' - f'Expected header template: {self.email_header_template_file}\n' - f'Expected body template: {self.email_body_template_file}' + f"Could not find one or more email templates needed by this action\n" + f"Expected header template: {self.email_header_template_file}\n" + f"Expected body template: {self.email_body_template_file}" ) - with open(self.email_body_template_file, 'r') as infile: + with open(self.email_body_template_file, "r") as infile: self.email_body_template = infile.read().strip() @classmethod def add_to_subparser(cls, subparsers): - action_parser = subparsers.add_parser( - cls._name, - description='Check the status of all pending DOI submissions.' - ) + action_parser = subparsers.add_parser(cls._name, description="Check the status of all pending DOI submissions.") action_parser.add_argument( - '-e', '--email', required=False, action='store_true', - help='If provided, the check action sends results to the default ' - 'recipients and pending DOI submitters.' + "-e", + "--email", + required=False, + action="store_true", + help="If provided, the check action sends results to the default " "recipients and pending DOI submitters.", ) action_parser.add_argument( - '-a', '--attachment', required=False, action='store_true', - help='If provided, the check action sends results as an email ' - 'attachment. Has no effect if --email is not also provided.' + "-a", + "--attachment", + required=False, + action="store_true", + help="If provided, the check action sends results as an email " + "attachment. Has no effect if --email is not also provided.", ) action_parser.add_argument( - '-s', '--submitter', required=False, metavar='"my.email@node.gov"', - help='The email address of the user to register as author of the ' - 'check action.' + "-s", + "--submitter", + required=False, + metavar='"my.email@node.gov"', + help="The email address of the user to register as author of the " "check action.", ) def _update_transaction_db(self, pending_record): @@ -113,15 +112,12 @@ def _update_transaction_db(self, pending_record): to the column names of the transaction database. """ - doi_value = pending_record['doi'] - identifier = pending_record['identifier'] + doi_value = pending_record["doi"] + identifier = pending_record["identifier"] - logger.info( - 'Checking release status for DOI %s (Identifier %s)', - doi_value, identifier - ) + logger.info("Checking release status for DOI %s (Identifier %s)", doi_value, identifier) - query_dict = {'doi': doi_value} + query_dict = {"doi": doi_value} doi_label = self._web_client.query_doi(query=query_dict) dois, errors = self._web_parser.parse_dois_from_label(doi_label) @@ -131,8 +127,7 @@ def _update_transaction_db(self, pending_record): doi = dois[0] if doi.status != DoiStatus.Pending: - logger.info("DOI has changed from status %s to %s", - DoiStatus.Pending, doi.status) + logger.info("DOI has changed from status %s to %s", DoiStatus.Pending, doi.status) # Set the author for this action doi.submitter = self._submitter @@ -141,41 +136,41 @@ def _update_transaction_db(self, pending_record): doi.previous_status = DoiStatus.Pending # Update the last updated time to mark the successful query - pending_record['date_updated'] = doi.date_record_updated.isoformat() + pending_record["date_updated"] = doi.date_record_updated.isoformat() # If there was a submission error, include the details. # Since we only check one DOI at a time, should be safe # to index by 0 here if errors: - doi.message = '\n'.join(errors[0]) + doi.message = "\n".join(errors[0]) # Log the update to the DOI entry transaction_obj = self.m_transaction_builder.prepare_transaction( - pending_record['node_id'], self._submitter, doi, - output_content_type=CONTENT_TYPE_JSON + pending_record["node_id"], self._submitter, doi, output_content_type=CONTENT_TYPE_JSON ) transaction_obj.log() else: - logger.info('No change in %s status for DOI %s (Identifier %s)', - DoiStatus.Pending, doi_value, identifier) + logger.info( + "No change in %s status for DOI %s (Identifier %s)", DoiStatus.Pending, doi_value, identifier + ) # Update the record we'll be using to populate the status email - pending_record['previous_status'] = pending_record['status'] - pending_record['status'] = doi.status - pending_record['identifier'] = identifier - pending_record['message'] = doi.message + pending_record["previous_status"] = pending_record["status"] + pending_record["status"] = doi.status + pending_record["identifier"] = identifier + pending_record["message"] = doi.message # Remove some behind-the-scenes fields users shouldn't care about - pending_record.pop('transaction_key', None) - pending_record.pop('is_latest', None) + pending_record.pop("transaction_key", None) + pending_record.pop("is_latest", None) else: - message = (f"No record for DOI {pending_record['doi']} " - f"(Identifier {identifier}) found at the service provider") - pending_record['message'] = message + message = ( + f"No record for DOI {pending_record['doi']} " f"(Identifier {identifier}) found at the service provider" + ) + pending_record["message"] = message logger.error(message) - def _get_distinct_nodes_and_submitters(self, i_check_result): """ Gets a list of distinct nodes and distinct submitters for each node from @@ -197,21 +192,19 @@ def _get_distinct_nodes_and_submitters(self, i_check_result): for one_result in i_check_result: # Make lowercase to be consistent. - node_id_key = one_result['node_id'].lower() + node_id_key = one_result["node_id"].lower() # Create an empty set of submitters and list of records keyed to # node_id_key if we haven't already. if node_id_key not in o_distinct_info: - o_distinct_info[node_id_key] = { - 'submitters': set(), 'records': list() - } + o_distinct_info[node_id_key] = {"submitters": set(), "records": list()} # Add the submitter to distinct_submitters for a particular node # if we haven't already. - o_distinct_info[node_id_key]['submitters'].add(one_result['submitter'].lower()) + o_distinct_info[node_id_key]["submitters"].add(one_result["submitter"].lower()) # Add each record to a particular node. - o_distinct_info[node_id_key]['records'].append(one_result) + o_distinct_info[node_id_key]["records"].append(one_result) return o_distinct_info @@ -233,19 +226,15 @@ def _prepare_attachment(self, i_dicts_per_submitter): """ # Convert a list of dict to JSON text to make it human readable. - attachment_text = json.dumps( - i_dicts_per_submitter, - indent=4 # Make output human readable by indentation - ) + attachment_text = json.dumps(i_dicts_per_submitter, indent=4) # Make output human readable by indentation # Add current time to make file unique - now_is = datetime.now().strftime('%Y%m%d-%H%M') - o_attachment_filename = f'doi_status_{now_is}.json' + now_is = datetime.now().strftime("%Y%m%d-%H%M") + o_attachment_filename = f"doi_status_{now_is}.json" o_attachment_part = MIMEMultipart() - part = MIMEBase('application', 'json') - part.add_header('Content-Disposition', - f'attachment; filename={o_attachment_filename}') + part = MIMEBase("application", "json") + part.add_header("Content-Disposition", f"attachment; filename={o_attachment_filename}") part.set_payload(attachment_text) o_attachment_part.attach(part) @@ -272,39 +261,35 @@ def _prepare_email_message(self, i_dicts_per_submitter): today = date.today() # Build the email header containing date and number of records - header_dict = { - 'my_date': today.strftime("%m/%d/%Y"), - 'my_records_count': len(i_dicts_per_submitter) - } - email_header = renderer.render_path( - self.email_header_template_file, header_dict - ) + header_dict = {"my_date": today.strftime("%m/%d/%Y"), "my_records_count": len(i_dicts_per_submitter)} + email_header = renderer.render_path(self.email_header_template_file, header_dict) # Build the email body containing the table of DOIs with status body_header = self.email_body_template.format( - record_index='#', id='ID', title='Title', doi='DOI', - identifier='PDS Identifier', previous_status='Previous Status', - status='Current Status' + record_index="#", + id="ID", + title="Title", + doi="DOI", + identifier="PDS Identifier", + previous_status="Previous Status", + status="Current Status", ) - body_divider = '-' * len(body_header) + body_divider = "-" * len(body_header) email_body = [body_header, body_divider] for index, record in enumerate(i_dicts_per_submitter): email_body.append( - self.email_body_template.format(record_index=index + 1, - id=record['doi'].split('/')[-1], - **record) + self.email_body_template.format(record_index=index + 1, id=record["doi"].split("/")[-1], **record) ) - email_body = '\n'.join(email_body) + email_body = "\n".join(email_body) o_email_entire_message = "\n".join([email_header, email_body]) logger.debug("o_email_entire_message:\n%s\n", o_email_entire_message) return o_email_entire_message - def _send_email(self, email_sender, final_receivers, subject_field, - email_entire_message, o_dicts_per_node): + def _send_email(self, email_sender, final_receivers, subject_field, email_entire_message, o_dicts_per_node): """ Sends an email containing a summary of the results of the last check action to the provided list of recipients. @@ -326,9 +311,7 @@ def _send_email(self, email_sender, final_receivers, subject_field, """ if not self._attachment: # This sends a brief email message. - self._emailer.sendmail( - email_sender, final_receivers, subject_field, email_entire_message - ) + self._emailer.sendmail(email_sender, final_receivers, subject_field, email_entire_message) else: # Try an alternative way to send the email so the attachment will be # viewable as an attachment in the email reader. @@ -360,18 +343,15 @@ def _group_updated_doi_records_and_email(self, i_check_result): """ # Get configurations related to sending email. - email_sender = self._config.get('OTHER', 'emailer_sender') - email_receivers_field = self._config.get('OTHER', 'emailer_receivers') + email_sender = self._config.get("OTHER", "emailer_sender") + email_receivers_field = self._config.get("OTHER", "emailer_receivers") # The receivers can be a comma-delimited list of addresses. - email_receivers_tokens = email_receivers_field.split(',') + email_receivers_tokens = email_receivers_field.split(",") # Get distinct list of email addresses from email_receivers_field in # case they have duplicates. - email_receivers = set( - [email_receiver_token.strip().lower() - for email_receiver_token in email_receivers_tokens] - ) + email_receivers = set([email_receiver_token.strip().lower() for email_receiver_token in email_receivers_tokens]) # Ensure the submitter of this check action included if they're not already email_receivers.add(self._submitter) @@ -389,34 +369,30 @@ def _group_updated_doi_records_and_email(self, i_check_result): final_receivers = deepcopy(email_receivers) # Add emails of all submitters for that node. - final_receivers |= o_distinct_info[node_key]['submitters'] + final_receivers |= o_distinct_info[node_key]["submitters"] now_is = datetime.now().isoformat() subject_field = f"DOI Submission Status Report For Node {node_key} On {now_is}" # Convert a list of dict to JSON text to make it human readable. - dois_per_node = [ - element['doi'] - for element in o_distinct_info[node_key]['records'] - ] + dois_per_node = [element["doi"] for element in o_distinct_info[node_key]["records"]] logger.debug( "NUM_RECORDS_PER_NODE_AND_SUBMITTERS: %s,%s,%d,%s", - node_key, dois_per_node, len(dois_per_node), - o_distinct_info[node_key]['submitters'] + node_key, + dois_per_node, + len(dois_per_node), + o_distinct_info[node_key]["submitters"], ) # Prepare the email message using all the dictionaries (records # associated with that node). - email_entire_message = self._prepare_email_message( - o_distinct_info[node_key]['records'] - ) + email_entire_message = self._prepare_email_message(o_distinct_info[node_key]["records"]) # Finally, send the email with all status changed per node. # The report is for a particular node, e.g 'img' send to all submitters # for that node (along with other recipients in final_receivers) self._send_email( - email_sender, final_receivers, subject_field, - email_entire_message, o_distinct_info[node_key]['records'] + email_sender, final_receivers, subject_field, email_entire_message, o_distinct_info[node_key]["records"] ) def run(self, **kwargs): @@ -438,8 +414,7 @@ def run(self, **kwargs): o_doi_list = self._list_obj.run(status=DoiStatus.Pending) pending_state_list = json.loads(o_doi_list) - logger.info(f'Found %d %s record(s) to check', len(pending_state_list), - DoiStatus.Pending) + logger.info(f"Found %d %s record(s) to check", len(pending_state_list), DoiStatus.Pending) if len(pending_state_list) > 0: for pending_record in pending_state_list: diff --git a/src/pds_doi_service/core/actions/draft.py b/src/pds_doi_service/core/actions/draft.py index d7f42f96..39c59a60 100644 --- a/src/pds_doi_service/core/actions/draft.py +++ b/src/pds_doi_service/core/actions/draft.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ======== draft.py @@ -12,22 +11,22 @@ Contains the definition for the Draft action of the Core PDS DOI Service. """ - import glob -from os.path import exists, join +from os.path import exists +from os.path import join from pds_doi_service.core.actions import DOICoreAction from pds_doi_service.core.actions.list import DOICoreActionList from pds_doi_service.core.entities.doi import DoiStatus -from pds_doi_service.core.input.exceptions import (DuplicatedTitleDOIException, - UnexpectedDOIActionException, - NoTransactionHistoryForIdentifierException, - TitleDoesNotMatchProductTypeException, - InputFormatException, - CriticalDOIException, - InvalidIdentifierException, - collect_exception_classes_and_messages, - raise_or_warn_exceptions) +from pds_doi_service.core.input.exceptions import collect_exception_classes_and_messages +from pds_doi_service.core.input.exceptions import CriticalDOIException +from pds_doi_service.core.input.exceptions import DuplicatedTitleDOIException +from pds_doi_service.core.input.exceptions import InputFormatException +from pds_doi_service.core.input.exceptions import InvalidIdentifierException +from pds_doi_service.core.input.exceptions import NoTransactionHistoryForIdentifierException +from pds_doi_service.core.input.exceptions import raise_or_warn_exceptions +from pds_doi_service.core.input.exceptions import TitleDoesNotMatchProductTypeException +from pds_doi_service.core.input.exceptions import UnexpectedDOIActionException from pds_doi_service.core.input.input_util import DOIInputUtil from pds_doi_service.core.input.node_util import NodeUtil from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON @@ -39,12 +38,12 @@ class DOICoreActionDraft(DOICoreAction): - _name = 'draft' - _description = 'Prepare a draft DOI record created from PDS4 labels.' + _name = "draft" + _description = "Prepare a draft DOI record created from PDS4 labels." _order = 10 - _run_arguments = ('input', 'node', 'submitter', 'lidvid', 'force', 'keywords') + _run_arguments = ("input", "node", "submitter", "lidvid", "force", "keywords") - DEFAULT_KEYWORDS = ['PDS', 'PDS4'] + DEFAULT_KEYWORDS = ["PDS", "PDS4"] """Default keywords added to each new draft request.""" def __init__(self, db_name=None): @@ -62,52 +61,66 @@ def __init__(self, db_name=None): self._lidvid = None self._force = False self._target = None - self._keywords = '' + self._keywords = "" @classmethod def add_to_subparser(cls, subparsers): action_parser = subparsers.add_parser( - cls._name, - description='Create a draft DOI record from existing PDS4 or DOI ' - 'labels.' + cls._name, description="Create a draft DOI record from existing PDS4 or DOI " "labels." ) node_values = NodeUtil.get_permissible_values() action_parser.add_argument( - '-i', '--input', required=False, - metavar='input/bundle_in_with_contributors.xml', - help='An input PDS4/DOI label. May be a local path or an HTTP address ' - 'resolving to a label file. Multiple inputs may be provided ' - 'via comma-delimited list. Must be provided if --lidvid is not ' - 'specified.' + "-i", + "--input", + required=False, + metavar="input/bundle_in_with_contributors.xml", + help="An input PDS4/DOI label. May be a local path or an HTTP address " + "resolving to a label file. Multiple inputs may be provided " + "via comma-delimited list. Must be provided if --lidvid is not " + "specified.", ) action_parser.add_argument( - '-n', '--node', required=True, metavar='"img"', - help='The PDS Discipline Node in charge of the DOI. Authorized ' - 'values are: ' + ','.join(node_values) + "-n", + "--node", + required=True, + metavar='"img"', + help="The PDS Discipline Node in charge of the DOI. Authorized " "values are: " + ",".join(node_values), ) action_parser.add_argument( - '-s', '--submitter', required=True, metavar='"my.email@node.gov"', - help='The email address to associate with the Draft record.' + "-s", + "--submitter", + required=True, + metavar='"my.email@node.gov"', + help="The email address to associate with the Draft record.", ) action_parser.add_argument( - '-l', '--lidvid', required=False, - metavar='urn:nasa:pds:lab_shocked_feldspars::1.0', - help='A LIDVID for an existing DOI record to move back to draft ' - 'status. Must be provided if --input is not specified.' + "-l", + "--lidvid", + required=False, + metavar="urn:nasa:pds:lab_shocked_feldspars::1.0", + help="A LIDVID for an existing DOI record to move back to draft " + "status. Must be provided if --input is not specified.", ) action_parser.add_argument( - '-f', '--force', required=False, action='store_true', - help='If provided, forces the action to proceed even if warnings are ' - 'encountered during submission of the draft record to the ' - 'database. Without this flag, any warnings encountered are ' - 'treated as fatal exceptions.', + "-f", + "--force", + required=False, + action="store_true", + help="If provided, forces the action to proceed even if warnings are " + "encountered during submission of the draft record to the " + "database. Without this flag, any warnings encountered are " + "treated as fatal exceptions.", ) action_parser.add_argument( - '-k', '--keywords', required=False, metavar='"Image"', default='', - help='Extra keywords to associate with the Draft record. Multiple ' - 'keywords must be separated by ",". Ignored when used with the ' - '--lidvid option.' + "-k", + "--keywords", + required=False, + metavar='"Image"', + default="", + help="Extra keywords to associate with the Draft record. Multiple " + 'keywords must be separated by ",". Ignored when used with the ' + "--lidvid option.", ) def _add_extra_keywords(self, keywords, io_doi): @@ -136,7 +149,7 @@ def _add_extra_keywords(self, keywords, io_doi): # The keywords are comma separated. The io_doi.keywords field is a set. if keywords: - keyword_tokens = set(map(str.strip, keywords.split(','))) + keyword_tokens = set(map(str.strip, keywords.split(","))) io_doi.keywords |= keyword_tokens @@ -162,12 +175,12 @@ def _transform_label_into_doi_objects(self, input_file, keywords): contains only a single DOI, a list of length 1 is returned. """ - input_util = DOIInputUtil(valid_extensions=['.json', '.xml']) + input_util = DOIInputUtil(valid_extensions=[".json", ".xml"]) o_dois = input_util.parse_dois_from_input_file(input_file) for o_doi in o_dois: - o_doi.publisher = self._config.get('OTHER', 'doi_publisher') + o_doi.publisher = self._config.get("OTHER", "doi_publisher") o_doi.contributor = NodeUtil().get_node_long_name(self._node) # Add 'status' field so the ranking in the workflow can be determined. @@ -208,8 +221,7 @@ def _run_single_file(self, input_file, node, submitter, force_flag, keywords): """ logger.info("Drafting input file %s", input_file) - logger.debug("node,submitter,force_flag,keywords: %s,%s,%s,%s", - node, submitter, force_flag, keywords) + logger.debug("node,submitter,force_flag,keywords: %s,%s,%s,%s", node, submitter, force_flag, keywords) exception_classes = [] exception_messages = [] @@ -223,12 +235,13 @@ def _run_single_file(self, input_file, node, submitter, force_flag, keywords): self._doi_validator.validate(doi) # Collect any exceptions/warnings for now and decide whether to # raise or log them later on - except (DuplicatedTitleDOIException, - InvalidIdentifierException, - UnexpectedDOIActionException, - TitleDoesNotMatchProductTypeException) as err: - (exception_classes, - exception_messages) = collect_exception_classes_and_messages( + except ( + DuplicatedTitleDOIException, + InvalidIdentifierException, + UnexpectedDOIActionException, + TitleDoesNotMatchProductTypeException, + ) as err: + (exception_classes, exception_messages) = collect_exception_classes_and_messages( err, exception_classes, exception_messages ) # Propagate input format exceptions, force flag should not affect @@ -244,15 +257,13 @@ def _run_single_file(self, input_file, node, submitter, force_flag, keywords): # WarningDOIException or log a warning with all the messages, # depending on the the state of the force flag if len(exception_classes) > 0: - raise_or_warn_exceptions(exception_classes, exception_messages, - log=force_flag) + raise_or_warn_exceptions(exception_classes, exception_messages, log=force_flag) for doi in dois: # Use TransactionBuilder to prepare all things related to writing to # the local transaction database. transaction = self.m_transaction_builder.prepare_transaction( - node, submitter, doi, input_path=input_file, - output_content_type=CONTENT_TYPE_JSON + node, submitter, doi, input_path=input_file, output_content_type=CONTENT_TYPE_JSON ) # Commit the transaction to the database @@ -288,22 +299,15 @@ def _draft_input_files(self, inputs): # The value of input can be a list of names, or a directory. # Split them up and let the input util library handle determination # of each type - list_of_inputs = inputs.split(',') + list_of_inputs = inputs.split(",") # Filter out any empty strings from trailing commas - list_of_inputs = list( - filter(lambda input_file: len(input_file), list_of_inputs) - ) + list_of_inputs = list(filter(lambda input_file: len(input_file), list_of_inputs)) # For each input file, transform the input into a list of in-memory # DOI objects, then concatenate to the master list of DOIs. for input_file in list_of_inputs: - dois.extend( - self._run_single_file( - input_file, self._node, self._submitter, self._force, - self._keywords - ) - ) + dois.extend(self._run_single_file(input_file, self._node, self._submitter, self._force, self._keywords)) if dois: # Create a single label containing records for each draft DOI @@ -312,9 +316,8 @@ def _draft_input_files(self, inputs): # Make sure were returning a valid label self._validator_service.validate(o_doi_label) else: - logger.warning('No DOI objects could be parsed from the provided ' - 'list of inputs: %s', list_of_inputs) - o_doi_label = '' + logger.warning("No DOI objects could be parsed from the provided " "list of inputs: %s", list_of_inputs) + o_doi_label = "" return o_doi_label @@ -349,22 +352,21 @@ def _set_lidvid_to_draft(self, lidvid): # Make sure we can locate the output label associated with this # transaction - transaction_location = transaction_record['transaction_key'] - label_files = glob.glob(join(transaction_location, 'output.*')) + transaction_location = transaction_record["transaction_key"] + label_files = glob.glob(join(transaction_location, "output.*")) if not label_files or not exists(label_files[0]): raise NoTransactionHistoryForIdentifierException( - f'Could not find a DOI label associated with LIDVID {lidvid}. ' - 'The database and transaction history location may be out of sync. ' - 'Please try resubmitting the record in reserve or draft.' + f"Could not find a DOI label associated with LIDVID {lidvid}. " + "The database and transaction history location may be out of sync. " + "Please try resubmitting the record in reserve or draft." ) label_file = label_files[0] # Label could contain entries for multiple LIDVIDs, so extract # just the one we care about - (lidvid_record, - content_type) = self._web_parser.get_record_for_identifier(label_file, lidvid) + (lidvid_record, content_type) = self._web_parser.get_record_for_identifier(label_file, lidvid) # Format label into an in-memory DOI object dois, _ = self._web_parser.parse_dois_from_label(lidvid_record, content_type) @@ -381,8 +383,7 @@ def _set_lidvid_to_draft(self, lidvid): # Re-commit transaction to official roll DOI back to draft status transaction = self.m_transaction_builder.prepare_transaction( - self._node, self._submitter, doi, input_path=label_file, - output_content_type=content_type + self._node, self._submitter, doi, input_path=label_file, output_content_type=content_type ) # Commit the transaction to the database @@ -413,8 +414,7 @@ def run(self, **kwargs): # Make sure we've been given something to work with if self._input is None and self._lidvid is None: - raise ValueError('A value must be provided for either --input or ' - '--lidvid when using the Draft action.') + raise ValueError("A value must be provided for either --input or " "--lidvid when using the Draft action.") if self._lidvid: return self._set_lidvid_to_draft(self._lidvid) diff --git a/src/pds_doi_service/core/actions/list.py b/src/pds_doi_service/core/actions/list.py index 4c217c25..20765031 100644 --- a/src/pds_doi_service/core/actions/list.py +++ b/src/pds_doi_service/core/actions/list.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ======= list.py @@ -12,11 +11,9 @@ Contains the definition for the List action of the Core PDS DOI Service. """ - import json from dateutil.parser import isoparse - from pds_doi_service.core.actions.action import DOICoreAction from pds_doi_service.core.db.doi_database import DOIDataBase from pds_doi_service.core.entities.doi import DoiStatus @@ -28,12 +25,10 @@ class DOICoreActionList(DOICoreAction): - _name = 'list' - _description = ('List DOI entries within the transaction database that match ' - 'the provided search criteria') + _name = "list" + _description = "List DOI entries within the transaction database that match " "the provided search criteria" _order = 40 - _run_arguments = ('doi', 'ids', 'node', 'status', 'start_update', - 'end_update', 'submitter') + _run_arguments = ("doi", "ids", "node", "status", "start_update", "end_update", "submitter") def __init__(self, db_name=None): super().__init__(db_name=db_name) @@ -43,66 +38,85 @@ def __init__(self, db_name=None): self.m_default_db_file = db_name else: # Default name of the database. - self.m_default_db_file = self._config.get('OTHER', 'db_file') + self.m_default_db_file = self._config.get("OTHER", "db_file") self._database_obj = DOIDataBase(self.m_default_db_file) @classmethod def add_to_subparser(cls, subparsers): action_parser = subparsers.add_parser( - cls._name, description='Extracts the submitted DOI from the local ' - 'transaction database using the following ' - 'selection criteria. Output is returned in ' - 'JSON format.' + cls._name, + description="Extracts the submitted DOI from the local " + "transaction database using the following " + "selection criteria. Output is returned in " + "JSON format.", ) node_values = NodeUtil.get_permissible_values() status_values = [status for status in DoiStatus] action_parser.add_argument( - '-n', '--node', required=False, metavar='"img,eng"', - help='A list of comma-separated node names to filter the available ' - 'DOI entries by. Valid values are: ' + ','.join(node_values) + "-n", + "--node", + required=False, + metavar='"img,eng"', + help="A list of comma-separated node names to filter the available " + "DOI entries by. Valid values are: " + ",".join(node_values), ) action_parser.add_argument( - '-status', '--status', required=False, metavar="draft,review", - help='A list of comma-separated submission status values to filter ' - 'the database query results by. Valid status values are: ' - '{}'.format(', '.join(status_values)) + "-status", + "--status", + required=False, + metavar="draft,review", + help="A list of comma-separated submission status values to filter " + "the database query results by. Valid status values are: " + "{}".format(", ".join(status_values)), ) action_parser.add_argument( - '-doi', '--doi', required=False, metavar='10.17189/21734', - help='A list of comma-delimited DOI values to use as filters with the ' - 'database query.' + "-doi", + "--doi", + required=False, + metavar="10.17189/21734", + help="A list of comma-delimited DOI values to use as filters with the " "database query.", ) action_parser.add_argument( - '-i', '--ids', required=False, - metavar='urn:nasa:pds:lab_shocked_feldspars', - help='A list of comma-delimited PDS identifiers to use as filters with ' - 'the database query. Each ID may contain one or more wildcards ' - '(*) to pattern match against.' + "-i", + "--ids", + required=False, + metavar="urn:nasa:pds:lab_shocked_feldspars", + help="A list of comma-delimited PDS identifiers to use as filters with " + "the database query. Each ID may contain one or more wildcards " + "(*) to pattern match against.", ) action_parser.add_argument( - '-start', '--start-update', required=False, - metavar='2020-01-01T19:02:15.000000', - help='The start time of the record update to use as a filter with the ' - 'database query. Should conform to a valid isoformat date string.' + "-start", + "--start-update", + required=False, + metavar="2020-01-01T19:02:15.000000", + help="The start time of the record update to use as a filter with the " + "database query. Should conform to a valid isoformat date string.", ) action_parser.add_argument( - '-end', '--end-update', required=False, - metavar='2020-12-311T23:59:00.000000', - help='The end time for record update time to use as a filter with the ' - 'database query. Should conform to a valid isoformat date string.' + "-end", + "--end-update", + required=False, + metavar="2020-12-311T23:59:00.000000", + help="The end time for record update time to use as a filter with the " + "database query. Should conform to a valid isoformat date string.", ) action_parser.add_argument( - '-s', '--submitter', required=False, metavar='"my.email@node.gov"', - help='A list of comma-separated email addresses to use as a filter ' - 'with the database query. Only entries containing the one of ' - 'the provided addresses as the submitter will be returned.' + "-s", + "--submitter", + required=False, + metavar='"my.email@node.gov"', + help="A list of comma-separated email addresses to use as a filter " + "with the database query. Only entries containing the one of " + "the provided addresses as the submitter will be returned.", ) - def parse_criteria(self, doi=None, ids=None, node=None, status=None, - start_update=None, end_update=None, submitter=None): + def parse_criteria( + self, doi=None, ids=None, node=None, status=None, start_update=None, end_update=None, submitter=None + ): """ Parse the command-line criteria into a dictionary format suitable for use to query the the local transaction database. @@ -136,25 +150,25 @@ def parse_criteria(self, doi=None, ids=None, node=None, status=None, query_criteria = {} if doi: - query_criteria['doi'] = doi.split(',') + query_criteria["doi"] = doi.split(",") if ids: - query_criteria['ids'] = ids.split(',') + query_criteria["ids"] = ids.split(",") if submitter: - query_criteria['submitter'] = submitter.split(',') + query_criteria["submitter"] = submitter.split(",") if node: - query_criteria['node'] = node.strip().split(',') + query_criteria["node"] = node.strip().split(",") if status: - query_criteria['status'] = status.strip().split(',') + query_criteria["status"] = status.strip().split(",") if start_update: - query_criteria['start_update'] = isoparse(start_update) + query_criteria["start_update"] = isoparse(start_update) if end_update: - query_criteria['end_update'] = isoparse(end_update) + query_criteria["end_update"] = isoparse(end_update) return query_criteria @@ -179,13 +193,11 @@ def transaction_for_identifier(self, identifier): provided identifier. """ - list_kwargs = {'ids': identifier} + list_kwargs = {"ids": identifier} list_results = json.loads(self.run(**list_kwargs)) if not list_results: - raise UnknownIdentifierException( - f'No record(s) could be found for identifier {identifier}.' - ) + raise UnknownIdentifierException(f"No record(s) could be found for identifier {identifier}.") # Latest record should be the only one returned record = list_results[0] @@ -218,7 +230,7 @@ def run(self, **kwargs): for row in rows: # Convert the datetime objects to iso8601 strings - for time_col in ('date_added', 'date_updated'): + for time_col in ("date_added", "date_updated"): row[columns.index(time_col)] = row[columns.index(time_col)].isoformat() result_json.append(dict(zip(columns, row))) diff --git a/src/pds_doi_service/core/actions/release.py b/src/pds_doi_service/core/actions/release.py index 2dae4c7d..7af0c30b 100644 --- a/src/pds_doi_service/core/actions/release.py +++ b/src/pds_doi_service/core/actions/release.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ========== release.py @@ -12,40 +11,41 @@ Contains the definition for the Release action of the Core PDS DOI Service. """ - from pds_doi_service.core.actions.action import DOICoreAction -from pds_doi_service.core.entities.doi import DoiEvent, DoiStatus -from pds_doi_service.core.input.exceptions import (InputFormatException, - DuplicatedTitleDOIException, - UnexpectedDOIActionException, - TitleDoesNotMatchProductTypeException, - SiteURLNotExistException, - CriticalDOIException, - InvalidIdentifierException, - collect_exception_classes_and_messages, - raise_or_warn_exceptions) +from pds_doi_service.core.entities.doi import DoiEvent +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.input.exceptions import collect_exception_classes_and_messages +from pds_doi_service.core.input.exceptions import CriticalDOIException +from pds_doi_service.core.input.exceptions import DuplicatedTitleDOIException +from pds_doi_service.core.input.exceptions import InputFormatException +from pds_doi_service.core.input.exceptions import InvalidIdentifierException +from pds_doi_service.core.input.exceptions import raise_or_warn_exceptions +from pds_doi_service.core.input.exceptions import SiteURLNotExistException +from pds_doi_service.core.input.exceptions import TitleDoesNotMatchProductTypeException +from pds_doi_service.core.input.exceptions import UnexpectedDOIActionException from pds_doi_service.core.input.input_util import DOIInputUtil from pds_doi_service.core.input.node_util import NodeUtil from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON from pds_doi_service.core.outputs.doi_validator import DOIValidator -from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE, DOIServiceFactory -from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST, WEB_METHOD_PUT +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE +from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST +from pds_doi_service.core.outputs.web_client import WEB_METHOD_PUT from pds_doi_service.core.util.general_util import get_logger logger = get_logger(__name__) class DOICoreActionRelease(DOICoreAction): - _name = 'release' - _description = ('Move a reserved DOI to review, or submit a DOI for ' - 'release to the service provider.') + _name = "release" + _description = "Move a reserved DOI to review, or submit a DOI for " "release to the service provider." _order = 20 - _run_arguments = ('input', 'node', 'submitter', 'force', 'no_review') + _run_arguments = ("input", "node", "submitter", "force", "no_review") def __init__(self, db_name=None): super().__init__(db_name=db_name) self._doi_validator = DOIValidator(db_name=db_name) - self._input_util = DOIInputUtil(valid_extensions=['.xml', '.json']) + self._input_util = DOIInputUtil(valid_extensions=[".xml", ".json"]) self._record_service = DOIServiceFactory.get_doi_record_service() self._validator_service = DOIServiceFactory.get_validator_service() self._web_client = DOIServiceFactory.get_web_client_service() @@ -60,40 +60,52 @@ def __init__(self, db_name=None): def add_to_subparser(cls, subparsers): action_parser = subparsers.add_parser( cls._name, - description='Release a DOI, in draft or reserve status, for review. ' - 'A DOI may also be released to the DOI service provider ' - 'directly.' + description="Release a DOI, in draft or reserve status, for review. " + "A DOI may also be released to the DOI service provider " + "directly.", ) action_parser.add_argument( - '-n', '--node', required=True, metavar='"img"', - help='The PDS Discipline Node in charge of the released DOI. ' - 'Authorized values are: {}' - .format(','.join(NodeUtil.get_permissible_values())) + "-n", + "--node", + required=True, + metavar='"img"', + help="The PDS Discipline Node in charge of the released DOI. " + "Authorized values are: {}".format(",".join(NodeUtil.get_permissible_values())), ) action_parser.add_argument( - '-f', '--force', required=False, action='store_true', - help='If provided, forces the release action to proceed even if ' - 'warning are encountered during submission of the release ' - 'request. Without this flag, any warnings encountered are ' - 'treated as fatal exceptions.' + "-f", + "--force", + required=False, + action="store_true", + help="If provided, forces the release action to proceed even if " + "warning are encountered during submission of the release " + "request. Without this flag, any warnings encountered are " + "treated as fatal exceptions.", ) action_parser.add_argument( - '-i', '--input', required=True, - metavar='input/DOI_Update_GEO_200318.xml', - help='A file containing a list of DOI metadata to update/release ' - 'in OSTI JSON/XML format (see https://www.osti.gov/iad2/docs#record-model).' - 'The input is produced by the Reserve and Draft actions, and ' - 'can be retrieved for a DOI with the List action.', + "-i", + "--input", + required=True, + metavar="input/DOI_Update_GEO_200318.xml", + help="A file containing a list of DOI metadata to update/release " + "in OSTI JSON/XML format (see https://www.osti.gov/iad2/docs#record-model)." + "The input is produced by the Reserve and Draft actions, and " + "can be retrieved for a DOI with the List action.", ) action_parser.add_argument( - '-s', '--submitter', required=True, metavar='"my.email@node.gov"', - help='The email address to associate with the Release request.' + "-s", + "--submitter", + required=True, + metavar='"my.email@node.gov"', + help="The email address to associate with the Release request.", ) action_parser.add_argument( - '--no-review', required=False, action='store_true', - help='If provided, the requested DOI will be released directly to ' - 'the DOI service provider for registration. Use to override the ' - 'default behavior of releasing a DOI to "review" status.' + "--no-review", + required=False, + action="store_true", + help="If provided, the requested DOI will be released directly to " + "the DOI service provider for registration. Use to override the " + 'default behavior of releasing a DOI to "review" status.', ) def _parse_input(self, input_file): @@ -132,7 +144,7 @@ def _complete_dois(self, dois): for doi in dois: # Make sure correct contributor and publisher fields are set doi.contributor = NodeUtil().get_node_long_name(self._node) - doi.publisher = self._config.get('OTHER', 'doi_publisher') + doi.publisher = self._config.get("OTHER", "doi_publisher") # Add 'status' field so the ranking in the workflow can be determined. doi.status = DoiStatus.Pending if self._no_review else DoiStatus.Review @@ -180,13 +192,14 @@ def _validate_dois(self, dois): # Validate the object representation of the DOI self._doi_validator.validate(doi) - except (DuplicatedTitleDOIException, - InvalidIdentifierException, - UnexpectedDOIActionException, - TitleDoesNotMatchProductTypeException, - SiteURLNotExistException) as err: - (exception_classes, - exception_messages) = collect_exception_classes_and_messages( + except ( + DuplicatedTitleDOIException, + InvalidIdentifierException, + UnexpectedDOIActionException, + TitleDoesNotMatchProductTypeException, + SiteURLNotExistException, + ) as err: + (exception_classes, exception_messages) = collect_exception_classes_and_messages( err, exception_classes, exception_messages ) @@ -194,8 +207,7 @@ def _validate_dois(self, dois): # WarningDOIException or log a warning with all the messages, # depending on the the state of the force flag if len(exception_classes) > 0: - raise_or_warn_exceptions(exception_classes, exception_messages, - log=self._force) + raise_or_warn_exceptions(exception_classes, exception_messages, log=self._force) return dois @@ -240,9 +252,7 @@ def run(self, **kwargs): for doi in dois: # Create a JSON format label to send to the service provider - io_doi_label = self._record_service.create_doi_record( - doi, content_type=CONTENT_TYPE_JSON - ) + io_doi_label = self._record_service.create_doi_record(doi, content_type=CONTENT_TYPE_JSON) # If the next step is to release, submit to the service provider and # use the response label for the local transaction database entry @@ -251,33 +261,25 @@ def run(self, **kwargs): # For OSTI, all submissions use the POST method # for DataCite, releasing a reserved DOI requires the PUT method - method = (WEB_METHOD_PUT - if service_type == SERVICE_TYPE_DATACITE - else WEB_METHOD_POST) + method = WEB_METHOD_PUT if service_type == SERVICE_TYPE_DATACITE else WEB_METHOD_POST # For DataCite, need to append the assigned DOI to the url # for the PUT request. For OSTI, can just default to the # url within the INI. if service_type == SERVICE_TYPE_DATACITE: - url = '{url}/{doi}'.format( - url=self._config.get('DATACITE', 'url'), doi=doi.doi - ) + url = "{url}/{doi}".format(url=self._config.get("DATACITE", "url"), doi=doi.doi) else: - url = self._config.get('OSTI', 'url') + url = self._config.get("OSTI", "url") doi, o_doi_label = self._web_client.submit_content( - url=url, - method=method, - payload=io_doi_label, - content_type=CONTENT_TYPE_JSON + url=url, method=method, payload=io_doi_label, content_type=CONTENT_TYPE_JSON ) # Otherwise, if the next step is review, the label we've already # created has marked all the Doi's as being the "review" step # so its ready to be submitted to the local transaction history transaction = self.m_transaction_builder.prepare_transaction( - self._node, self._submitter, doi, input_path=self._input, - output_content_type=CONTENT_TYPE_JSON + self._node, self._submitter, doi, input_path=self._input, output_content_type=CONTENT_TYPE_JSON ) # Commit the transaction to the local database @@ -298,8 +300,6 @@ def run(self, **kwargs): # Create the return output label containing records for all submitted DOI's # Note this action always returns JSON format to ensure interoperability # between the potential service providers - output_label = self._record_service.create_doi_record( - output_dois, content_type=CONTENT_TYPE_JSON - ) + output_label = self._record_service.create_doi_record(output_dois, content_type=CONTENT_TYPE_JSON) return output_label diff --git a/src/pds_doi_service/core/actions/reserve.py b/src/pds_doi_service/core/actions/reserve.py index 3ef553bc..b1543683 100644 --- a/src/pds_doi_service/core/actions/reserve.py +++ b/src/pds_doi_service/core/actions/reserve.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ========== reserve.py @@ -12,35 +11,39 @@ Contains the definition for the Reserve action of the Core PDS DOI Service. """ - from pds_doi_service.core.actions.action import DOICoreAction -from pds_doi_service.core.entities.doi import DoiEvent, DoiStatus -from pds_doi_service.core.input.exceptions import (CriticalDOIException, - DuplicatedTitleDOIException, - InputFormatException, - InvalidIdentifierException, - SiteURLNotExistException, - TitleDoesNotMatchProductTypeException, - UnexpectedDOIActionException, - collect_exception_classes_and_messages, - raise_or_warn_exceptions) +from pds_doi_service.core.entities.doi import DoiEvent +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.input.exceptions import collect_exception_classes_and_messages +from pds_doi_service.core.input.exceptions import CriticalDOIException +from pds_doi_service.core.input.exceptions import DuplicatedTitleDOIException +from pds_doi_service.core.input.exceptions import InputFormatException +from pds_doi_service.core.input.exceptions import InvalidIdentifierException +from pds_doi_service.core.input.exceptions import raise_or_warn_exceptions +from pds_doi_service.core.input.exceptions import SiteURLNotExistException +from pds_doi_service.core.input.exceptions import TitleDoesNotMatchProductTypeException +from pds_doi_service.core.input.exceptions import UnexpectedDOIActionException from pds_doi_service.core.input.input_util import DOIInputUtil from pds_doi_service.core.input.node_util import NodeUtil from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON from pds_doi_service.core.outputs.doi_validator import DOIValidator -from pds_doi_service.core.outputs.service import DOIServiceFactory, SERVICE_TYPE_DATACITE -from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST, WEB_METHOD_PUT +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE +from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST +from pds_doi_service.core.outputs.web_client import WEB_METHOD_PUT from pds_doi_service.core.util.general_util import get_logger logger = get_logger(__name__) class DOICoreActionReserve(DOICoreAction): - _name = 'reserve' - _description = ("Submit a request to reserve a DOI prior to public release. " - "Reserved DOI's may be released after via the release action.") + _name = "reserve" + _description = ( + "Submit a request to reserve a DOI prior to public release. " + "Reserved DOI's may be released after via the release action." + ) _order = 0 - _run_arguments = ('input', 'node', 'submitter', 'dry_run', 'force') + _run_arguments = ("input", "node", "submitter", "dry_run", "force") def __init__(self, db_name=None): super().__init__(db_name=db_name) @@ -59,39 +62,52 @@ def __init__(self, db_name=None): @classmethod def add_to_subparser(cls, subparsers): action_parser = subparsers.add_parser( - cls._name, description='Create a DOI for one or more unpublished datasets. ' - 'The input is a spreadsheet or CSV file ' - 'containing records to reserve DOIs for.' + cls._name, + description="Create a DOI for one or more unpublished datasets. " + "The input is a spreadsheet or CSV file " + "containing records to reserve DOIs for.", ) action_parser.add_argument( - '-n', '--node', required=True, metavar='"img"', + "-n", + "--node", + required=True, + metavar='"img"', help="The PDS Discipline Node in charge of the submission of the DOI. " - "Authorized values are: {}" - .format(','.join(NodeUtil.get_permissible_values())) + "Authorized values are: {}".format(",".join(NodeUtil.get_permissible_values())), ) action_parser.add_argument( - '-f', '--force', required=False, action='store_true', - help='If provided, forces the reserve action to proceed even if ' - 'warnings are encountered during submission of the reserve ' - 'request. Without this flag, any warnings encountered are ' - 'treated as fatal exceptions.' + "-f", + "--force", + required=False, + action="store_true", + help="If provided, forces the reserve action to proceed even if " + "warnings are encountered during submission of the reserve " + "request. Without this flag, any warnings encountered are " + "treated as fatal exceptions.", ) action_parser.add_argument( - '-i', '--input', required=True, - metavar='input/DOI_Reserved_GEO_200318.csv', - help='A PDS4 XML label, OSTI XML/JSON label or XLS/CSV ' - 'spreadsheet file with the following columns: ' + - ','.join(DOIInputUtil.MANDATORY_COLUMNS) + "-i", + "--input", + required=True, + metavar="input/DOI_Reserved_GEO_200318.csv", + help="A PDS4 XML label, OSTI XML/JSON label or XLS/CSV " + "spreadsheet file with the following columns: " + ",".join(DOIInputUtil.MANDATORY_COLUMNS), ) action_parser.add_argument( - '-s', '--submitter', required=True, metavar='"my.email@node.gov"', - help='The email address to associate with the Reserve request.' + "-s", + "--submitter", + required=True, + metavar='"my.email@node.gov"', + help="The email address to associate with the Reserve request.", ) action_parser.add_argument( - '-dry-run', '--dry-run', required=False, action='store_true', + "-dry-run", + "--dry-run", + required=False, + action="store_true", help="Performs the Reserve request without submitting the record. " - "The record is logged to the local database with a status " - "of 'reserved_not_submitted'." + "The record is logged to the local database with a status " + "of 'reserved_not_submitted'.", ) def _parse_input(self, input_file): @@ -131,7 +147,7 @@ def _complete_dois(self, dois): # First set contributor, publisher at the beginning of the function # to ensure that they are set in case of an exception. doi.contributor = NodeUtil().get_node_long_name(self._node) - doi.publisher = self._config.get('OTHER', 'doi_publisher') + doi.publisher = self._config.get("OTHER", "doi_publisher") # Add 'status' field so the ranking in the workflow can be determined doi.status = DoiStatus.Reserved_not_submitted if self._dry_run else DoiStatus.Reserved @@ -182,13 +198,14 @@ def _validate_dois(self, dois): self._doi_validator.validate(doi) # Collect all warnings and exceptions so they can be combined into # a single WarningDOIException - except (DuplicatedTitleDOIException, - InvalidIdentifierException, - UnexpectedDOIActionException, - TitleDoesNotMatchProductTypeException, - SiteURLNotExistException) as err: - (exception_classes, - exception_messages) = collect_exception_classes_and_messages( + except ( + DuplicatedTitleDOIException, + InvalidIdentifierException, + UnexpectedDOIActionException, + TitleDoesNotMatchProductTypeException, + SiteURLNotExistException, + ) as err: + (exception_classes, exception_messages) = collect_exception_classes_and_messages( err, exception_classes, exception_messages ) @@ -196,8 +213,7 @@ def _validate_dois(self, dois): # WarningDOIException or log a warning with all the messages, # depending on the the state of the force flag if len(exception_classes) > 0: - raise_or_warn_exceptions(exception_classes, exception_messages, - log=self._force) + raise_or_warn_exceptions(exception_classes, exception_messages, log=self._force) return dois @@ -235,9 +251,7 @@ def run(self, **kwargs): for doi in dois: # Create the JSON request label to send - io_doi_label = self._record_service.create_doi_record( - doi, content_type=CONTENT_TYPE_JSON - ) + io_doi_label = self._record_service.create_doi_record(doi, content_type=CONTENT_TYPE_JSON) # Submit the Reserve request if this isn't a dry run # Note that for both OSTI and DataCite, reserve requests should @@ -249,26 +263,20 @@ def run(self, **kwargs): # we need to use a PUT request on the URL associated to the DOI if service_type == SERVICE_TYPE_DATACITE and doi.doi: method = WEB_METHOD_PUT - url = '{url}/{doi}'.format( - url=self._config.get('DATACITE', 'url'), doi=doi.doi - ) + url = "{url}/{doi}".format(url=self._config.get("DATACITE", "url"), doi=doi.doi) # Otherwise, for both DataCite and OSTI, just a POST request # on the default endpoint is sufficient else: method = WEB_METHOD_POST - url = self._config.get(service_type.upper(), 'url') + url = self._config.get(service_type.upper(), "url") doi, o_doi_label = self._web_client.submit_content( - method=method, - url=url, - payload=io_doi_label, - content_type=CONTENT_TYPE_JSON + method=method, url=url, payload=io_doi_label, content_type=CONTENT_TYPE_JSON ) # Log the inputs and outputs of this transaction transaction = self.m_transaction_builder.prepare_transaction( - self._node, self._submitter, doi, input_path=self._input, - output_content_type=CONTENT_TYPE_JSON + self._node, self._submitter, doi, input_path=self._input, output_content_type=CONTENT_TYPE_JSON ) # Commit the transaction to the local database @@ -290,8 +298,6 @@ def run(self, **kwargs): # Create the return output label containing records for all submitted DOI's # Note this action always returns JSON format to ensure interoperability # between the potential service providers - output_label = self._record_service.create_doi_record( - output_dois, content_type=CONTENT_TYPE_JSON - ) + output_label = self._record_service.create_doi_record(output_dois, content_type=CONTENT_TYPE_JSON) return output_label diff --git a/src/pds_doi_service/core/actions/test/__init__.py b/src/pds_doi_service/core/actions/test/__init__.py index ca07f807..ec545673 100644 --- a/src/pds_doi_service/core/actions/test/__init__.py +++ b/src/pds_doi_service/core/actions/test/__init__.py @@ -1,12 +1,14 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for core actions -''' - - +""" import unittest -from . import check_test, draft_test, list_test, release_test, reserve_test + +from . import check_test +from . import draft_test +from . import list_test +from . import release_test +from . import reserve_test def suite(): diff --git a/src/pds_doi_service/core/actions/test/check_test.py b/src/pds_doi_service/core/actions/test/check_test.py index e73ca2da..8fc3e38e 100644 --- a/src/pds_doi_service/core/actions/test/check_test.py +++ b/src/pds_doi_service/core/actions/test/check_test.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - import configparser import datetime import os @@ -8,53 +7,62 @@ import tempfile import time import unittest - from email import message_from_bytes -from os.path import abspath, dirname, join -from pkg_resources import resource_filename +from os.path import abspath +from os.path import dirname +from os.path import join from unittest.mock import patch import pds_doi_service.core.outputs.datacite.datacite_web_client import pds_doi_service.core.outputs.osti.osti_web_client from pds_doi_service.core.actions import DOICoreActionCheck from pds_doi_service.core.db.doi_database import DOIDataBase -from pds_doi_service.core.entities.doi import DoiStatus, ProductType +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML -from pds_doi_service.core.outputs.service import (DOIServiceFactory, - SERVICE_TYPE_OSTI, - SERVICE_TYPE_DATACITE) +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE +from pds_doi_service.core.outputs.service import SERVICE_TYPE_OSTI from pds_doi_service.core.util.config_parser import DOIConfigUtil +from pkg_resources import resource_filename class CheckActionTestCase(unittest.TestCase): - test_dir = resource_filename(__name__, '') - input_dir = abspath( - join(test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + test_dir = resource_filename(__name__, "") + input_dir = abspath(join(test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) @classmethod def setUp(cls): - cls.db_name = join(cls.test_dir, 'doi_temp.db') + cls.db_name = join(cls.test_dir, "doi_temp.db") cls._database_obj = DOIDataBase(cls.db_name) # Write a record with new doi into temporary database - doi = '10.17189/29348' - lid = 'urn:nasa:pds:lab_shocked_feldspars' - vid = '1.0' - identifier = lid + '::' + vid - transaction_key = './transaction_history/img/2020-06-15T18:42:45.653317' + doi = "10.17189/29348" + lid = "urn:nasa:pds:lab_shocked_feldspars" + vid = "1.0" + identifier = lid + "::" + vid + transaction_key = "./transaction_history/img/2020-06-15T18:42:45.653317" date_added = datetime.datetime.now() date_updated = datetime.datetime.now() status = DoiStatus.Pending - title = 'Laboratory Shocked Feldspars Bundle' + title = "Laboratory Shocked Feldspars Bundle" product_type = ProductType.Collection - product_type_specific = 'PDS4 Collection' - discipline_node = 'img' - submitter = 'img-submitter@jpl.nasa.gov' + product_type_specific = "PDS4 Collection" + discipline_node = "img" + submitter = "img-submitter@jpl.nasa.gov" cls._database_obj.write_doi_info_to_database( - identifier, transaction_key, doi, date_added, date_updated, status, - title, product_type, product_type_specific, submitter, discipline_node + identifier, + transaction_key, + doi, + date_added, + date_updated, + status, + title, + product_type, + product_type_specific, + submitter, + discipline_node, ) # Create the check action and assign it our temp database @@ -65,8 +73,9 @@ def tearDown(cls): if os.path.exists(cls.db_name): os.remove(cls.db_name) - def webclient_query_patch_nominal(self, query, url=None, username=None, - password=None, content_type=CONTENT_TYPE_XML): + def webclient_query_patch_nominal( + self, query, url=None, username=None, password=None, content_type=CONTENT_TYPE_XML + ): """ Patch for DOIWebClient.query_doi(). @@ -79,19 +88,16 @@ def webclient_query_patch_nominal(self, query, url=None, username=None, # Read an output label that corresponds to the DOI we're # checking for, and that has a status of 'registered' or 'findable' if DOIServiceFactory.get_service_type() == SERVICE_TYPE_OSTI: - label = join(CheckActionTestCase.input_dir, - 'DOI_Release_20200727_from_register.xml') + label = join(CheckActionTestCase.input_dir, "DOI_Release_20200727_from_register.xml") else: - label = join(CheckActionTestCase.input_dir, - 'DOI_Release_20210615_from_release.json') + label = join(CheckActionTestCase.input_dir, "DOI_Release_20210615_from_release.json") - with open(label, 'r') as infile: + with open(label, "r") as infile: label_contents = infile.read() return label_contents - def webclient_query_patch_error(self, query, url=None, username=None, - password=None, content_type=CONTENT_TYPE_XML): + def webclient_query_patch_error(self, query, url=None, username=None, password=None, content_type=CONTENT_TYPE_XML): """ Patch for DOIWebClient.query_doi(). @@ -103,14 +109,14 @@ def webclient_query_patch_error(self, query, url=None, username=None, """ # Read an output label that corresponds to the DOI we're # checking for, and that has a status of 'error' - with open(join(CheckActionTestCase.input_dir, - 'DOI_Release_20200727_from_error.xml'), 'r') as infile: + with open(join(CheckActionTestCase.input_dir, "DOI_Release_20200727_from_error.xml"), "r") as infile: xml_contents = infile.read() return xml_contents - def webclient_query_patch_no_change(self, query, url=None, username=None, - password=None, content_type=CONTENT_TYPE_XML): + def webclient_query_patch_no_change( + self, query, url=None, username=None, password=None, content_type=CONTENT_TYPE_XML + ): """ Patch for DOIOstiWebClient.query_doi(). @@ -121,18 +127,19 @@ def webclient_query_patch_no_change(self, query, url=None, username=None, """ # Read an output label that corresponds to the DOI we're # checking for, and that has a status of 'pending' - with open(join(CheckActionTestCase.input_dir, - 'DOI_Release_20200727_from_release.xml'), 'r') as infile: + with open(join(CheckActionTestCase.input_dir, "DOI_Release_20200727_from_release.xml"), "r") as infile: xml_contents = infile.read() return xml_contents @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'query_doi', webclient_query_patch_nominal) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "query_doi", webclient_query_patch_nominal + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'query_doi', webclient_query_patch_nominal) + "query_doi", + webclient_query_patch_nominal, + ) def test_check_for_pending_entries(self): """Test check action that returns a successfully registered entry""" pending_records = self._action.run(email=False) @@ -141,17 +148,18 @@ def test_check_for_pending_entries(self): pending_record = pending_records[0] - self.assertEqual(pending_record['previous_status'], DoiStatus.Pending) - self.assertIn(pending_record['status'], (DoiStatus.Registered, DoiStatus.Findable)) - self.assertEqual(pending_record['submitter'], 'img-submitter@jpl.nasa.gov') - self.assertEqual(pending_record['doi'], '10.17189/29348') - self.assertEqual(pending_record['identifier'], 'urn:nasa:pds:lab_shocked_feldspars::1.0') + self.assertEqual(pending_record["previous_status"], DoiStatus.Pending) + self.assertIn(pending_record["status"], (DoiStatus.Registered, DoiStatus.Findable)) + self.assertEqual(pending_record["submitter"], "img-submitter@jpl.nasa.gov") + self.assertEqual(pending_record["doi"], "10.17189/29348") + self.assertEqual(pending_record["identifier"], "urn:nasa:pds:lab_shocked_feldspars::1.0") - @unittest.skipIf(DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE, - "DataCite does not return errors via label") + @unittest.skipIf( + DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE, "DataCite does not return errors via label" + ) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'query_doi', webclient_query_patch_error) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "query_doi", webclient_query_patch_error + ) def test_check_for_pending_entries_w_error(self): """Test check action that returns an error result""" pending_records = self._action.run(email=False) @@ -160,20 +168,22 @@ def test_check_for_pending_entries_w_error(self): pending_record = pending_records[0] - self.assertEqual(pending_record['previous_status'], DoiStatus.Pending) - self.assertEqual(pending_record['status'], DoiStatus.Error) - self.assertEqual(pending_record['submitter'], 'img-submitter@jpl.nasa.gov') - self.assertEqual(pending_record['doi'], '10.17189/29348') - self.assertEqual(pending_record['identifier'], 'urn:nasa:pds:lab_shocked_feldspars::1.0') + self.assertEqual(pending_record["previous_status"], DoiStatus.Pending) + self.assertEqual(pending_record["status"], DoiStatus.Error) + self.assertEqual(pending_record["submitter"], "img-submitter@jpl.nasa.gov") + self.assertEqual(pending_record["doi"], "10.17189/29348") + self.assertEqual(pending_record["identifier"], "urn:nasa:pds:lab_shocked_feldspars::1.0") # There should be a message to go along with the error - self.assertIsNotNone(pending_record['message']) + self.assertIsNotNone(pending_record["message"]) - @unittest.skipIf(DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE, - "DataCite does not assign a pending state to release requests") + @unittest.skipIf( + DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE, + "DataCite does not assign a pending state to release requests", + ) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'query_doi', webclient_query_patch_no_change) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "query_doi", webclient_query_patch_no_change + ) def test_check_for_pending_entries_w_no_change(self): """Test check action when no pending entries have been updated""" pending_records = self._action.run(email=False) @@ -182,11 +192,11 @@ def test_check_for_pending_entries_w_no_change(self): pending_record = pending_records[0] - self.assertEqual(pending_record['previous_status'], DoiStatus.Pending) - self.assertEqual(pending_record['status'], DoiStatus.Pending) - self.assertEqual(pending_record['submitter'], 'img-submitter@jpl.nasa.gov') - self.assertEqual(pending_record['doi'], '10.17189/29348') - self.assertEqual(pending_record['identifier'], 'urn:nasa:pds:lab_shocked_feldspars::1.0') + self.assertEqual(pending_record["previous_status"], DoiStatus.Pending) + self.assertEqual(pending_record["status"], DoiStatus.Pending) + self.assertEqual(pending_record["submitter"], "img-submitter@jpl.nasa.gov") + self.assertEqual(pending_record["doi"], "10.17189/29348") + self.assertEqual(pending_record["identifier"], "urn:nasa:pds:lab_shocked_feldspars::1.0") def get_config_patch(self): """ @@ -196,28 +206,26 @@ def get_config_patch(self): parser = configparser.ConfigParser() # default configuration - conf_default = 'conf.ini.default' - conf_default_path = abspath( - join(dirname(__file__), os.pardir, os.pardir, 'util', conf_default) - ) + conf_default = "conf.ini.default" + conf_default_path = abspath(join(dirname(__file__), os.pardir, os.pardir, "util", conf_default)) parser.read(conf_default_path) - parser['OTHER']['emailer_local_host'] = 'localhost' - parser['OTHER']['emailer_port'] = '1025' + parser["OTHER"]["emailer_local_host"] = "localhost" + parser["OTHER"]["emailer_port"] = "1025" parser = DOIConfigUtil._resolve_relative_path(parser) return parser + @patch.object(pds_doi_service.core.util.config_parser.DOIConfigUtil, "get_config", get_config_patch) @patch.object( - pds_doi_service.core.util.config_parser.DOIConfigUtil, - 'get_config', get_config_patch) - @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'query_doi', webclient_query_patch_nominal) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "query_doi", webclient_query_patch_nominal + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'query_doi', webclient_query_patch_nominal) + "query_doi", + webclient_query_patch_nominal, + ) def test_email_receipt(self): """Test sending of the check action status via email""" # Create a new check action so our patched config is pulled in @@ -228,8 +236,7 @@ def test_email_receipt(self): # By default, all this server is does is echo email payloads to # standard out, so provide a temp file to capture it debug_email_proc = subprocess.Popen( - ['python', '-u', '-m', 'smtpd', '-n', '-c', 'DebuggingServer', 'localhost:1025'], - stdout=temp_file + ["python", "-u", "-m", "smtpd", "-n", "-c", "DebuggingServer", "localhost:1025"], stdout=temp_file ) # Give the debug smtp server a chance to start listening @@ -237,8 +244,7 @@ def test_email_receipt(self): try: # Run the check action and have it send an email w/ attachment - action.run(email=True, attachment=True, - submitter='email-test@email.com') + action.run(email=True, attachment=True, submitter="email-test@email.com") # Read the raw email contents (payload) from the subprocess # into a string @@ -254,14 +260,14 @@ def test_email_receipt(self): # made it in # Email address provided to check action should be present - self.assertIn('email-test@email.com', message) + self.assertIn("email-test@email.com", message) # Subject line should be present - self.assertIn('DOI Submission Status Report For Node', message) + self.assertIn("DOI Submission Status Report For Node", message) # Attachment should also be provided - self.assertIn('Content-Disposition: attachment; filename=doi_status_', message) + self.assertIn("Content-Disposition: attachment; filename=doi_status_", message) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/actions/test/draft_test.py b/src/pds_doi_service/core/actions/test/draft_test.py index d9cd44e7..79725029 100644 --- a/src/pds_doi_service/core/actions/test/draft_test.py +++ b/src/pds_doi_service/core/actions/test/draft_test.py @@ -1,29 +1,29 @@ #!/usr/bin/env python - -from datetime import datetime import os -from os.path import abspath, join -import unittest import tempfile - -from pkg_resources import resource_filename +import unittest +from datetime import datetime +from os.path import abspath +from os.path import join from pds_doi_service.core.actions.draft import DOICoreActionDraft from pds_doi_service.core.actions.release import DOICoreActionRelease -from pds_doi_service.core.entities.doi import DoiStatus, ProductType -from pds_doi_service.core.input.exceptions import InputFormatException, WarningDOIException +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.input.exceptions import InputFormatException +from pds_doi_service.core.input.exceptions import WarningDOIException from pds_doi_service.core.outputs.service import DOIServiceFactory +from pkg_resources import resource_filename + class DraftActionTestCase(unittest.TestCase): # Because validation has been added to each action, the force=True is # required for each test as the command line is not parsed. def setUp(self): - self.test_dir = resource_filename(__name__, '') - self.input_dir = abspath( - join(self.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) - self.db_name = join(self.test_dir, 'doi_temp.db') + self.test_dir = resource_filename(__name__, "") + self.input_dir = abspath(join(self.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) + self.db_name = join(self.test_dir, "doi_temp.db") self._draft_action = DOICoreActionDraft(db_name=self.db_name) self._review_action = DOICoreActionRelease(db_name=self.db_name) @@ -37,10 +37,10 @@ def tearDown(self): def test_local_dir_one_file(self): """Test draft request with local dir containing one file""" kwargs = { - 'input': join(self.input_dir, 'draft_dir_one_file'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "draft_dir_one_file"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -55,7 +55,7 @@ def test_local_dir_one_file(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.editors), 3) self.assertEqual(len(doi.keywords), 18) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:insight_cameras::1.0') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras::1.0") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -64,10 +64,10 @@ def test_local_dir_one_file(self): def test_local_dir_two_files(self): """Test draft request with local dir containing two files""" kwargs = { - 'input': join(self.input_dir, 'draft_dir_two_files'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "draft_dir_two_files"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -84,12 +84,12 @@ def test_local_dir_two_files(self): self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) self.assertIsInstance(doi.date_record_added, datetime) - self.assertTrue(doi.related_identifier.startswith('urn:nasa:pds:insight_cameras::1')) - self.assertTrue(doi.title.startswith('InSight Cameras Bundle 1.')) + self.assertTrue(doi.related_identifier.startswith("urn:nasa:pds:insight_cameras::1")) + self.assertTrue(doi.title.startswith("InSight Cameras Bundle 1.")) # Make sure for the "bundle_in_with_contributors.xml" file, we # parsed the editors - if doi.related_identifier == 'urn:nasa:pds:insight_cameras::1.0': + if doi.related_identifier == "urn:nasa:pds:insight_cameras::1.0": self.assertEqual(len(doi.editors), 3) # For "bundle_in.xml", there should be no editors else: @@ -98,10 +98,10 @@ def test_local_dir_two_files(self): def test_local_pds4_bundle(self): """Test draft request with a local bundle path""" kwargs = { - 'input': join(self.input_dir, 'bundle_in_with_contributors.xml'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "bundle_in_with_contributors.xml"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -116,7 +116,7 @@ def test_local_pds4_bundle(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.editors), 3) self.assertEqual(len(doi.keywords), 18) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:insight_cameras::1.0') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras::1.0") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -125,10 +125,10 @@ def test_local_pds4_bundle(self): def test_remote_pds4_bundle(self): """Test draft request with a remote bundle URL""" kwargs = { - 'input': 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/bundle.xml', - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/bundle.xml", + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -142,7 +142,7 @@ def test_remote_pds4_bundle(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.keywords), 18) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:insight_cameras::1.0') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras::1.0") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -151,10 +151,10 @@ def test_remote_pds4_bundle(self): def test_local_osti_label(self): """Test draft action with a local OSTI label""" kwargs = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_review.xml'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "DOI_Release_20200727_from_review.xml"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -171,20 +171,20 @@ def test_local_osti_label(self): def test_local_unsupported_file(self): """Attempt a draft with a unsupported file types""" kwargs = { - 'input': join(self.input_dir, 'DOI_Reserved_GEO_200318.csv'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "DOI_Reserved_GEO_200318.csv"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } with self.assertRaises(InputFormatException): self._draft_action.run(**kwargs) kwargs = { - 'input': join(self.input_dir, 'DOI_Reserved_GEO_200318.xlsx'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "DOI_Reserved_GEO_200318.xlsx"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } with self.assertRaises(InputFormatException): @@ -193,10 +193,10 @@ def test_local_unsupported_file(self): def test_remote_collection(self): """Test draft request with a remote collection URL""" kwargs = { - 'input': 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/data/collection_data.xml', - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/data/collection_data.xml", + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -210,7 +210,7 @@ def test_remote_collection(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.keywords), 12) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:insight_cameras:data::1.0') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras:data::1.0") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -219,10 +219,10 @@ def test_remote_collection(self): def test_remote_browse_collection(self): """Test draft request with a remote browse collection URL""" kwargs = { - 'input': 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/browse/collection_browse.xml', - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/browse/collection_browse.xml", + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -236,8 +236,8 @@ def test_remote_browse_collection(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.keywords), 12) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:insight_cameras:browse::1.0') - self.assertEqual(doi.description, 'Collection of BROWSE products.') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras:browse::1.0") + self.assertEqual(doi.description, "Collection of BROWSE products.") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -246,10 +246,10 @@ def test_remote_browse_collection(self): def test_remote_calibration_collection(self): """Test draft request with remote calibration collection URL""" kwargs = { - 'input': 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/calibration/collection_calibration.xml', - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/calibration/collection_calibration.xml", + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -263,10 +263,8 @@ def test_remote_calibration_collection(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.keywords), 14) - self.assertEqual(doi.related_identifier, - 'urn:nasa:pds:insight_cameras:calibration::1.0') - self.assertEqual(doi.description, - 'Collection of CALIBRATION files/products to include in the archive.') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras:calibration::1.0") + self.assertEqual(doi.description, "Collection of CALIBRATION files/products to include in the archive.") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -275,10 +273,10 @@ def test_remote_calibration_collection(self): def test_remote_document_collection(self): """Test draft request with remote document collection URL""" kwargs = { - 'input': 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/document/collection_document.xml', - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/document/collection_document.xml", + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**kwargs) @@ -292,8 +290,8 @@ def test_remote_document_collection(self): self.assertEqual(len(doi.authors), 4) self.assertEqual(len(doi.keywords), 12) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:insight_cameras:document::1.0') - self.assertEqual(doi.description, 'Collection of DOCUMENT products.') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras:document::1.0") + self.assertEqual(doi.description, "Collection of DOCUMENT products.") self.assertEqual(doi.status, DoiStatus.Draft) self.assertEqual(doi.product_type, ProductType.Dataset) self.assertIsInstance(doi.publication_date, datetime) @@ -303,10 +301,10 @@ def test_move_lidvid_to_draft(self): """Test moving a review record back to draft via its lidvid""" # Start by drafting a PDS label draft_kwargs = { - 'input': join(self.input_dir, 'bundle_in_with_contributors.xml'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "bundle_in_with_contributors.xml"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } draft_doi_label = self._draft_action.run(**draft_kwargs) @@ -318,24 +316,17 @@ def test_move_lidvid_to_draft(self): self.assertEqual(dois[0].status, DoiStatus.Draft) # Move the draft to review - json_doi_label = self._record_service.create_doi_record(dois, content_type='json') + json_doi_label = self._record_service.create_doi_record(dois, content_type="json") - with tempfile.NamedTemporaryFile(mode='w', dir=self.test_dir, suffix='.json') as outfile: + with tempfile.NamedTemporaryFile(mode="w", dir=self.test_dir, suffix=".json") as outfile: outfile.write(json_doi_label) outfile.flush() - review_kwargs = { - 'input': outfile.name, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True - } + review_kwargs = {"input": outfile.name, "node": "img", "submitter": "my_user@my_node.gov", "force": True} review_doi_label = self._review_action.run(**review_kwargs) - dois, errors = self._web_parser.parse_dois_from_label( - review_doi_label, content_type='json' - ) + dois, errors = self._web_parser.parse_dois_from_label(review_doi_label, content_type="json") self.assertEqual(len(dois), 1) self.assertEqual(len(errors), 0) @@ -345,17 +336,15 @@ def test_move_lidvid_to_draft(self): # Finally, move the review record back to draft with the lidvid option draft_kwargs = { - 'lidvid': doi.related_identifier, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "lidvid": doi.related_identifier, + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } draft_doi_label = self._draft_action.run(**draft_kwargs) - dois, errors = self._web_parser.parse_dois_from_label( - draft_doi_label, content_type='json' - ) + dois, errors = self._web_parser.parse_dois_from_label(draft_doi_label, content_type="json") self.assertEqual(len(dois), 1) self.assertEqual(len(errors), 0) @@ -367,10 +356,10 @@ def test_force_flag(self): submitting a draft. """ draft_kwargs = { - 'input': join(self.input_dir, 'bundle_in_with_contributors.xml'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "bundle_in_with_contributors.xml"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } draft_doi_label = self._draft_action.run(**draft_kwargs) @@ -384,20 +373,15 @@ def test_force_flag(self): doi = dois[0] # Slightly modify the lidvid so we trigger the "duplicate title" warning - doi.related_identifier += '.1' + doi.related_identifier += ".1" - modified_draft_label = self._record_service.create_doi_record(doi, content_type='json') + modified_draft_label = self._record_service.create_doi_record(doi, content_type="json") - with tempfile.NamedTemporaryFile(mode='w', dir=self.test_dir, suffix='.json') as outfile: + with tempfile.NamedTemporaryFile(mode="w", dir=self.test_dir, suffix=".json") as outfile: outfile.write(modified_draft_label) outfile.flush() - draft_kwargs = { - 'input': outfile.name, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': False - } + draft_kwargs = {"input": outfile.name, "node": "img", "submitter": "my_user@my_node.gov", "force": False} # Should get a warning exception containing the duplicate title finding with self.assertRaises(WarningDOIException): @@ -405,12 +389,7 @@ def test_force_flag(self): # Now try again with the force flag set and we should bypass the # warning - draft_kwargs = { - 'input': outfile.name, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True - } + draft_kwargs = {"input": outfile.name, "node": "img", "submitter": "my_user@my_node.gov", "force": True} try: self._draft_action.run(**draft_kwargs) @@ -418,5 +397,5 @@ def test_force_flag(self): self.fail() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/actions/test/list_test.py b/src/pds_doi_service/core/actions/test/list_test.py index e906260f..72096779 100644 --- a/src/pds_doi_service/core/actions/test/list_test.py +++ b/src/pds_doi_service/core/actions/test/list_test.py @@ -1,29 +1,26 @@ #!/usr/bin/env python - import json import os -from os.path import abspath, join -import unittest import tempfile - -from pkg_resources import resource_filename +import unittest +from os.path import abspath +from os.path import join from pds_doi_service.core.actions.draft import DOICoreActionDraft from pds_doi_service.core.actions.list import DOICoreActionList from pds_doi_service.core.actions.release import DOICoreActionRelease from pds_doi_service.core.entities.doi import DoiStatus from pds_doi_service.core.outputs.service import DOIServiceFactory +from pkg_resources import resource_filename class ListActionTestCase(unittest.TestCase): # TODO: add additional unit tests for other list query parameters def setUp(self): - self.test_dir = resource_filename(__name__, '') - self.input_dir = abspath( - join(self.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) - self.db_name = join(self.test_dir, 'doi_temp.db') + self.test_dir = resource_filename(__name__, "") + self.input_dir = abspath(join(self.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) + self.db_name = join(self.test_dir, "doi_temp.db") self._list_action = DOICoreActionList(db_name=self.db_name) self._draft_action = DOICoreActionDraft(db_name=self.db_name) self._release_action = DOICoreActionRelease(db_name=self.db_name) @@ -38,10 +35,10 @@ def test_list_by_status(self): """Test listing of entries, querying by workflow status""" # Submit a draft, then query by draft status to retrieve draft_kwargs = { - 'input': join(self.input_dir, 'bundle_in_with_contributors.xml'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True + "input": join(self.input_dir, "bundle_in_with_contributors.xml"), + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, } doi_label = self._draft_action.run(**draft_kwargs) @@ -49,70 +46,60 @@ def test_list_by_status(self): dois, _ = self._web_parser.parse_dois_from_label(doi_label) doi = dois[0] - list_kwargs = { - 'status': DoiStatus.Draft - } + list_kwargs = {"status": DoiStatus.Draft} list_result = json.loads(self._list_action.run(**list_kwargs)) self.assertEqual(len(list_result), 1) list_result = list_result[0] - self.assertEqual(list_result['status'], doi.status) - self.assertEqual(list_result['title'], doi.title) - self.assertEqual(list_result['subtype'], doi.product_type_specific) - self.assertEqual(list_result['identifier'], doi.related_identifier) + self.assertEqual(list_result["status"], doi.status) + self.assertEqual(list_result["title"], doi.title) + self.assertEqual(list_result["subtype"], doi.product_type_specific) + self.assertEqual(list_result["identifier"], doi.related_identifier) # Now move the draft to review, use JSON as the format to ensure # this test works for both DataCite and OSTI - doi_label = self._record_service.create_doi_record( - dois, content_type='json' - ) + doi_label = self._record_service.create_doi_record(dois, content_type="json") - with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as temp_file: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json") as temp_file: temp_file.write(doi_label) temp_file.flush() review_kwargs = { - 'input': temp_file.name, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'force': True, - 'no_review': False + "input": temp_file.name, + "node": "img", + "submitter": "my_user@my_node.gov", + "force": True, + "no_review": False, } review_json = self._release_action.run(**review_kwargs) - dois, _ = self._web_parser.parse_dois_from_label( - review_json, content_type='json' - ) + dois, _ = self._web_parser.parse_dois_from_label(review_json, content_type="json") doi = dois[0] # Now query for review status - list_kwargs = { - 'status': DoiStatus.Review - } + list_kwargs = {"status": DoiStatus.Review} list_result = json.loads(self._list_action.run(**list_kwargs)) self.assertEqual(len(list_result), 1) list_result = list_result[0] - self.assertEqual(list_result['status'], doi.status) - self.assertEqual(list_result['title'], doi.title) - self.assertEqual(list_result['subtype'], doi.product_type_specific) - self.assertEqual(list_result['identifier'], doi.related_identifier) + self.assertEqual(list_result["status"], doi.status) + self.assertEqual(list_result["title"], doi.title) + self.assertEqual(list_result["subtype"], doi.product_type_specific) + self.assertEqual(list_result["identifier"], doi.related_identifier) # Finally, query for draft status again, should get no results back - list_kwargs = { - 'status': DoiStatus.Draft - } + list_kwargs = {"status": DoiStatus.Draft} list_result = json.loads(self._list_action.run(**list_kwargs)) self.assertEqual(len(list_result), 0) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/actions/test/release_test.py b/src/pds_doi_service/core/actions/test/release_test.py index a985128e..1ac37b09 100644 --- a/src/pds_doi_service/core/actions/test/release_test.py +++ b/src/pds_doi_service/core/actions/test/release_test.py @@ -1,19 +1,19 @@ #!/usr/bin/env python - import os -from os.path import abspath, join import unittest +from os.path import abspath +from os.path import join from unittest.mock import patch -from pkg_resources import resource_filename - import pds_doi_service.core.outputs.datacite.datacite_web_client import pds_doi_service.core.outputs.osti.osti_web_client from pds_doi_service.core.actions.release import DOICoreActionRelease from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML from pds_doi_service.core.outputs.service import DOIServiceFactory -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML, CONTENT_TYPE_JSON from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST +from pkg_resources import resource_filename class ReleaseActionTestCase(unittest.TestCase): @@ -22,11 +22,9 @@ class ReleaseActionTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) - cls.db_name = 'doi_temp.db' + cls.test_dir = resource_filename(__name__, "") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) + cls.db_name = "doi_temp.db" # Remove db_name if exist to have a fresh start otherwise exception will be # raised about using existing lidvid. @@ -42,9 +40,9 @@ def tearDownClass(cls): if os.path.isfile(cls.db_name): os.remove(cls.db_name) - def webclient_submit_patch(self, payload, url=None, username=None, - password=None, method=WEB_METHOD_POST, - content_type=CONTENT_TYPE_XML): + def webclient_submit_patch( + self, payload, url=None, username=None, password=None, method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML + ): """ Patch for DOIWebClient.submit_content(). @@ -53,17 +51,13 @@ def webclient_submit_patch(self, payload, url=None, username=None, """ # Parse the DOI's from the input label, update status to 'pending', # and create the output label - dois, _ = ReleaseActionTestCase._web_parser.parse_dois_from_label( - payload, content_type=CONTENT_TYPE_JSON - ) + dois, _ = ReleaseActionTestCase._web_parser.parse_dois_from_label(payload, content_type=CONTENT_TYPE_JSON) doi = dois[0] doi.status = DoiStatus.Pending - o_doi_label = ReleaseActionTestCase._record_service.create_doi_record( - doi, content_type=CONTENT_TYPE_JSON - ) + o_doi_label = ReleaseActionTestCase._record_service.create_doi_record(doi, content_type=CONTENT_TYPE_JSON) return doi, o_doi_label @@ -84,9 +78,7 @@ def run_release_test(self, release_args, expected_dois, expected_status): """ o_doi_label = self._release_action.run(**release_args) - dois, errors = self._web_parser.parse_dois_from_label( - o_doi_label, content_type=CONTENT_TYPE_JSON - ) + dois, errors = self._web_parser.parse_dois_from_label(o_doi_label, content_type=CONTENT_TYPE_JSON) # Should get the expected number of parsed DOI's self.assertEqual(len(dois), expected_dois) @@ -102,73 +94,69 @@ def test_reserve_release_to_review(self): """Test release to review status with a reserved DOI entry""" release_args = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_reserve.xml'), - 'node': 'img', - 'submitter': 'img-submitter@jpl.nasa.gov', - 'force': True, - 'no_review': False + "input": join(self.input_dir, "DOI_Release_20200727_from_reserve.xml"), + "node": "img", + "submitter": "img-submitter@jpl.nasa.gov", + "force": True, + "no_review": False, } - self.run_release_test( - release_args, expected_dois=1, expected_status=DoiStatus.Review - ) + self.run_release_test(release_args, expected_dois=1, expected_status=DoiStatus.Review) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'submit_content', webclient_submit_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "submit_content", webclient_submit_patch + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'submit_content', webclient_submit_patch) + "submit_content", + webclient_submit_patch, + ) def test_reserve_release_to_provider(self): """Test release directly to the service provider with a reserved DOI entry""" release_args = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_reserve.xml'), - 'node': 'img', - 'submitter': 'img-submitter@jpl.nasa.gov', - 'force': True, - 'no_review': True + "input": join(self.input_dir, "DOI_Release_20200727_from_reserve.xml"), + "node": "img", + "submitter": "img-submitter@jpl.nasa.gov", + "force": True, + "no_review": True, } - self.run_release_test( - release_args, expected_dois=1, expected_status=DoiStatus.Pending - ) + self.run_release_test(release_args, expected_dois=1, expected_status=DoiStatus.Pending) def test_draft_release_to_review(self): """Test release to review status with a draft DOI entry""" release_args = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_draft.xml'), - 'node': 'img', - 'submitter': 'img-submitter@jpl.nasa.gov', - 'force': True, - 'no_review': False + "input": join(self.input_dir, "DOI_Release_20200727_from_draft.xml"), + "node": "img", + "submitter": "img-submitter@jpl.nasa.gov", + "force": True, + "no_review": False, } - self.run_release_test( - release_args, expected_dois=1, expected_status=DoiStatus.Review - ) + self.run_release_test(release_args, expected_dois=1, expected_status=DoiStatus.Review) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'submit_content', webclient_submit_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "submit_content", webclient_submit_patch + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'submit_content', webclient_submit_patch) + "submit_content", + webclient_submit_patch, + ) def test_draft_release_to_provider(self): """Test release directly to the service provider with a draft DOI entry""" release_args = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_draft.xml'), - 'node': 'img', - 'submitter': 'img-submitter@jpl.nasa.gov', - 'force': True, - 'no_review': True + "input": join(self.input_dir, "DOI_Release_20200727_from_draft.xml"), + "node": "img", + "submitter": "img-submitter@jpl.nasa.gov", + "force": True, + "no_review": True, } - self.run_release_test( - release_args, expected_dois=1, expected_status=DoiStatus.Pending - ) + self.run_release_test(release_args, expected_dois=1, expected_status=DoiStatus.Pending) def test_review_release_to_review(self): """ @@ -178,38 +166,36 @@ def test_review_release_to_review(self): """ release_args = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_review.xml'), - 'node': 'img', - 'submitter': 'img-submitter@jpl.nasa.gov', - 'force': True, - 'no_review': False + "input": join(self.input_dir, "DOI_Release_20200727_from_review.xml"), + "node": "img", + "submitter": "img-submitter@jpl.nasa.gov", + "force": True, + "no_review": False, } - self.run_release_test( - release_args, expected_dois=1, expected_status=DoiStatus.Review - ) + self.run_release_test(release_args, expected_dois=1, expected_status=DoiStatus.Review) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'submit_content', webclient_submit_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "submit_content", webclient_submit_patch + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'submit_content', webclient_submit_patch) + "submit_content", + webclient_submit_patch, + ) def test_review_release_to_osti(self): """Test release directly to the service provider with a review DOI entry""" release_args = { - 'input': join(self.input_dir, 'DOI_Release_20200727_from_review.xml'), - 'node': 'img', - 'submitter': 'img-submitter@jpl.nasa.gov', - 'force': True, - 'no_review': True + "input": join(self.input_dir, "DOI_Release_20200727_from_review.xml"), + "node": "img", + "submitter": "img-submitter@jpl.nasa.gov", + "force": True, + "no_review": True, } - self.run_release_test( - release_args, expected_dois=1, expected_status=DoiStatus.Pending - ) + self.run_release_test(release_args, expected_dois=1, expected_status=DoiStatus.Pending) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/actions/test/reserve_test.py b/src/pds_doi_service/core/actions/test/reserve_test.py index 7791f210..7994807b 100644 --- a/src/pds_doi_service/core/actions/test/reserve_test.py +++ b/src/pds_doi_service/core/actions/test/reserve_test.py @@ -1,19 +1,20 @@ #!/usr/bin/env python - import os -from os.path import abspath, join import unittest +from os.path import abspath +from os.path import join from unittest.mock import patch -from pkg_resources import resource_filename - import pds_doi_service.core.outputs.datacite.datacite_web_client import pds_doi_service.core.outputs.osti.osti_web_client from pds_doi_service.core.actions.reserve import DOICoreActionReserve from pds_doi_service.core.entities.doi import DoiStatus -from pds_doi_service.core.outputs.service import DOIServiceFactory, SERVICE_TYPE_OSTI -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML, CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_OSTI from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST +from pkg_resources import resource_filename class ReserveActionTestCase(unittest.TestCase): @@ -22,11 +23,9 @@ class ReserveActionTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) - cls.db_name = 'doi_temp.db' + cls.test_dir = resource_filename(__name__, "") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) + cls.db_name = "doi_temp.db" # Remove db_name if exist to have a fresh start otherwise exception will be # raised about using existing lidvid. @@ -42,9 +41,9 @@ def tearDownClass(cls): if os.path.isfile(cls.db_name): os.remove(cls.db_name) - def webclient_submit_patch(self, payload, url=None, username=None, - password=None, method=WEB_METHOD_POST, - content_type=CONTENT_TYPE_XML): + def webclient_submit_patch( + self, payload, url=None, username=None, password=None, method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML + ): """ Patch for DOIWebClient.submit_content(). @@ -53,17 +52,13 @@ def webclient_submit_patch(self, payload, url=None, username=None, """ # Parse the DOI's from the input label, update status to 'reserved', # and create the output label - dois, _ = ReserveActionTestCase._web_parser.parse_dois_from_label( - payload, content_type=CONTENT_TYPE_JSON - ) + dois, _ = ReserveActionTestCase._web_parser.parse_dois_from_label(payload, content_type=CONTENT_TYPE_JSON) doi = dois[0] doi.status = DoiStatus.Reserved - o_doi_label = ReserveActionTestCase._record_service.create_doi_record( - doi, content_type=CONTENT_TYPE_JSON - ) + o_doi_label = ReserveActionTestCase._record_service.create_doi_record(doi, content_type=CONTENT_TYPE_JSON) return doi, o_doi_label @@ -84,9 +79,7 @@ def run_reserve_test(self, reserve_args, expected_dois, expected_status): """ o_doi_label = self._reserve_action.run(**reserve_args) - dois, errors = self._web_parser.parse_dois_from_label( - o_doi_label, content_type=CONTENT_TYPE_JSON - ) + dois, errors = self._web_parser.parse_dois_from_label(o_doi_label, content_type=CONTENT_TYPE_JSON) # Should get the expected number of parsed DOI's self.assertEqual(len(dois), expected_dois) @@ -104,41 +97,37 @@ def test_reserve_xlsx_dry_run(self): dry run flag to avoid submission. """ reserve_args = { - 'input': join(self.input_dir, - 'DOI_Reserved_GEO_200318_with_corrected_identifier.xlsx'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'dry_run': True, - 'force': True + "input": join(self.input_dir, "DOI_Reserved_GEO_200318_with_corrected_identifier.xlsx"), + "node": "img", + "submitter": "my_user@my_node.gov", + "dry_run": True, + "force": True, } - self.run_reserve_test( - reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved_not_submitted - ) + self.run_reserve_test(reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved_not_submitted) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'submit_content', webclient_submit_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "submit_content", webclient_submit_patch + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'submit_content', webclient_submit_patch) + "submit_content", + webclient_submit_patch, + ) def test_reserve_xlsx_and_submit(self): """ Test Reserve action with a local excel spreadsheet, submitting the result to the service provider. """ reserve_args = { - 'input': join(self.input_dir, - 'DOI_Reserved_GEO_200318_with_corrected_identifier.xlsx'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'dry_run': False, - 'force': True + "input": join(self.input_dir, "DOI_Reserved_GEO_200318_with_corrected_identifier.xlsx"), + "node": "img", + "submitter": "my_user@my_node.gov", + "dry_run": False, + "force": True, } - self.run_reserve_test( - reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved - ) + self.run_reserve_test(reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved) def test_reserve_csv_dry_run(self): """ @@ -146,39 +135,37 @@ def test_reserve_csv_dry_run(self): to avoid submission to the service provider. """ reserve_args = { - 'input': join(self.input_dir, 'DOI_Reserved_GEO_200318.csv'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'dry_run': True, - 'force': True + "input": join(self.input_dir, "DOI_Reserved_GEO_200318.csv"), + "node": "img", + "submitter": "my_user@my_node.gov", + "dry_run": True, + "force": True, } - self.run_reserve_test( - reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved_not_submitted - ) + self.run_reserve_test(reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved_not_submitted) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'submit_content', webclient_submit_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "submit_content", webclient_submit_patch + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'submit_content', webclient_submit_patch) + "submit_content", + webclient_submit_patch, + ) def test_reserve_csv_and_submit(self): """ Test Reserve action with a local CSV file, submitting the result to the service provider. """ reserve_args = { - 'input': join(self.input_dir, 'DOI_Reserved_GEO_200318.csv'), - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'dry_run': False, - 'force': True + "input": join(self.input_dir, "DOI_Reserved_GEO_200318.csv"), + "node": "img", + "submitter": "my_user@my_node.gov", + "dry_run": False, + "force": True, } - self.run_reserve_test( - reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved - ) + self.run_reserve_test(reserve_args, expected_dois=3, expected_status=DoiStatus.Reserved) def test_reserve_json_dry_run(self): """ @@ -188,28 +175,28 @@ def test_reserve_json_dry_run(self): # Select the appropriate JSON format based on the currently configured # service if DOIServiceFactory.get_service_type() == SERVICE_TYPE_OSTI: - input_file = join(self.input_dir, 'DOI_Release_20210216_from_reserve.json') + input_file = join(self.input_dir, "DOI_Release_20210216_from_reserve.json") else: - input_file = join(self.input_dir, 'DOI_Release_20210615_from_reserve.json') + input_file = join(self.input_dir, "DOI_Release_20210615_from_reserve.json") reserve_args = { - 'input': input_file, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'dry_run': True, - 'force': True + "input": input_file, + "node": "img", + "submitter": "my_user@my_node.gov", + "dry_run": True, + "force": True, } - self.run_reserve_test( - reserve_args, expected_dois=1, expected_status=DoiStatus.Reserved_not_submitted - ) + self.run_reserve_test(reserve_args, expected_dois=1, expected_status=DoiStatus.Reserved_not_submitted) @patch.object( - pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, - 'submit_content', webclient_submit_patch) + pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "submit_content", webclient_submit_patch + ) @patch.object( pds_doi_service.core.outputs.datacite.datacite_web_client.DOIDataCiteWebClient, - 'submit_content', webclient_submit_patch) + "submit_content", + webclient_submit_patch, + ) def test_reserve_json_and_submit(self): """ Test Reserve action with a local JSON file, submitting the result to @@ -218,22 +205,20 @@ def test_reserve_json_and_submit(self): # Select the appropriate JSON format based on the currently configured # service if DOIServiceFactory.get_service_type() == SERVICE_TYPE_OSTI: - input_file = join(self.input_dir, 'DOI_Release_20210216_from_reserve.json') + input_file = join(self.input_dir, "DOI_Release_20210216_from_reserve.json") else: - input_file = join(self.input_dir, 'DOI_Release_20210615_from_reserve.json') + input_file = join(self.input_dir, "DOI_Release_20210615_from_reserve.json") reserve_args = { - 'input': input_file, - 'node': 'img', - 'submitter': 'my_user@my_node.gov', - 'dry_run': False, - 'force': True + "input": input_file, + "node": "img", + "submitter": "my_user@my_node.gov", + "dry_run": False, + "force": True, } - self.run_reserve_test( - reserve_args, expected_dois=1, expected_status=DoiStatus.Reserved - ) + self.run_reserve_test(reserve_args, expected_dois=1, expected_status=DoiStatus.Reserved) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/cmd/pds_doi_cmd.py b/src/pds_doi_service/core/cmd/pds_doi_cmd.py index c6792089..6fc75b11 100755 --- a/src/pds_doi_service/core/cmd/pds_doi_cmd.py +++ b/src/pds_doi_service/core/cmd/pds_doi_cmd.py @@ -5,7 +5,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============== pds_doi_cmd.py @@ -13,12 +12,11 @@ Contains the main function for the pds_doi_cmd.py script. """ - import importlib import os -from pds_doi_service.core.util.general_util import get_logger from pds_doi_service.core.actions.action import DOICoreAction +from pds_doi_service.core.util.general_util import get_logger logger = get_logger(__name__) @@ -31,19 +29,19 @@ def main(): # Moved many argument parsing to each action class. logger.info(f"run_dir {os.getcwd()}") - module = importlib.import_module(f'pds_doi_service.core.actions.{action_type}') - action_class = getattr(module, f'DOICoreAction{action_type.capitalize()}') + module = importlib.import_module(f"pds_doi_service.core.actions.{action_type}") + action_class = getattr(module, f"DOICoreAction{action_type.capitalize()}") action = action_class() # Convert the argparse.Namespace to a dictionary that we can feed in as kwargs kwargs = vars(arguments) # No action subclasses should be expecting subcommand, so remove it here - kwargs.pop('subcommand', None) + kwargs.pop("subcommand", None) output = action.run(**kwargs) print(output) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/pds_doi_service/core/db/doi_database.py b/src/pds_doi_service/core/db/doi_database.py index a1163b56..963f9c63 100644 --- a/src/pds_doi_service/core/db/doi_database.py +++ b/src/pds_doi_service/core/db/doi_database.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ =============== doi_database.py @@ -13,16 +12,17 @@ Contains classes and functions for interfacing with the local transaction database (SQLite3). """ - import datetime -from collections import OrderedDict -from datetime import datetime, timezone, timedelta - import sqlite3 +from collections import OrderedDict +from datetime import datetime +from datetime import timedelta +from datetime import timezone from sqlite3 import Error +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.util.config_parser import DOIConfigUtil -from pds_doi_service.core.entities.doi import DoiStatus, ProductType from pds_doi_service.core.util.general_util import get_logger # Get the common logger and set the level for this file. @@ -34,20 +34,21 @@ class DOIDataBase: Provides a mechanism to write, update and read rows to/from a local SQLite3 database. """ + DOI_DB_SCHEMA = OrderedDict( { - 'identifier': 'TEXT NOT NULL', # PDS identifier (any version) - 'doi': 'TEXT', # DOI (may be null for pending or draft) - 'status': 'TEXT NOT NULL', # current status - 'title': 'TEXT', # title used for the DOI - 'submitter': 'TEXT', # email of the submitter of the DOI - 'type': 'TEXT', # product type - 'subtype': 'TEXT', # subtype of the product - 'node_id': 'TEXT NOT NULL', # steward discipline node ID - 'date_added': 'INT', # as Unix epoch seconds - 'date_updated': 'INT NOT NULL', # as Unix epoch seconds - 'transaction_key': 'TEXT NOT NULL', # transaction (key is node id/datetime) - 'is_latest': 'BOOLEAN' # whether the transaction is the latest + "identifier": "TEXT NOT NULL", # PDS identifier (any version) + "doi": "TEXT", # DOI (may be null for pending or draft) + "status": "TEXT NOT NULL", # current status + "title": "TEXT", # title used for the DOI + "submitter": "TEXT", # email of the submitter of the DOI + "type": "TEXT", # product type + "subtype": "TEXT", # subtype of the product + "node_id": "TEXT NOT NULL", # steward discipline node ID + "date_added": "INT", # as Unix epoch seconds + "date_updated": "INT NOT NULL", # as Unix epoch seconds + "transaction_key": "TEXT NOT NULL", # transaction (key is node id/datetime) + "is_latest": "BOOLEAN", # whether the transaction is the latest } ) """ @@ -62,7 +63,7 @@ class DOIDataBase: def __init__(self, db_file): self._config = DOIConfigUtil().get_config() self.m_database_name = db_file - self.m_default_table_name = 'doi' + self.m_default_table_name = "doi" self.m_my_conn = None def get_database_name(self): @@ -79,20 +80,15 @@ def close_database(self): # Set m_my_conn to None to signify that there is no connection. self.m_my_conn = None else: - logger.warn("Database connection to %s has not been started or is " - "already closed", self.m_database_name) + logger.warn("Database connection to %s has not been started or is " "already closed", self.m_database_name) def create_connection(self): """Create and return a connection to the SQLite database.""" if self.m_my_conn is not None: - logger.warning("There is already an open database connection, " - "closing existing connection.") + logger.warning("There is already an open database connection, " "closing existing connection.") self.close_database() - logger.info( - "Connecting to SQLite3 (ver %s) database %s", - sqlite3.version, self.m_database_name - ) + logger.info("Connecting to SQLite3 (ver %s) database %s", sqlite3.version, self.m_database_name) try: self.m_my_conn = sqlite3.connect(self.m_database_name) @@ -129,26 +125,22 @@ def check_if_table_exists(self, table_name): o_table_exists_flag = False if self.m_my_conn is None: - logger.warn("Not connected to %s, establishing new connection...", - self.m_database_name) + logger.warn("Not connected to %s, establishing new connection...", self.m_database_name) self.create_connection() table_pointer = self.m_my_conn.cursor() # Get the count of tables with the given name. - query_string = ( - "SELECT count(name) FROM sqlite_master WHERE type='table' AND " - f"name='{table_name}'" - ) + query_string = "SELECT count(name) FROM sqlite_master WHERE type='table' AND " f"name='{table_name}'" - logger.info('Executing query: %s', query_string) + logger.info("Executing query: %s", query_string) table_pointer.execute(query_string) # If the count is 1, then table exists. if table_pointer.fetchone()[0] == 1: o_table_exists_flag = True - logger.debug('o_table_exists_flag: %s', o_table_exists_flag) + logger.debug("o_table_exists_flag: %s", o_table_exists_flag) return o_table_exists_flag @@ -156,7 +148,7 @@ def drop_table(self, table_name): """Delete the given table from the SQLite database.""" if self.m_my_conn: logger.debug("Executing query: DROP TABLE %s", table_name) - self.m_my_conn.execute(f'DROP TABLE {table_name}') + self.m_my_conn.execute(f"DROP TABLE {table_name}") def query_string_for_table_creation(self, table_name): """ @@ -175,16 +167,16 @@ def query_string_for_table_creation(self, table_name): the database. """ - o_query_string = f'CREATE TABLE {table_name} ' - o_query_string += '(' + o_query_string = f"CREATE TABLE {table_name} " + o_query_string += "(" for index, (column, constraints) in enumerate(self.DOI_DB_SCHEMA.items()): - o_query_string += f'{column} {constraints}' + o_query_string += f"{column} {constraints}" if index < (self.EXPECTED_NUM_COLS - 1): - o_query_string += ',' + o_query_string += "," - o_query_string += ');' + o_query_string += ");" logger.debug("CREATE o_query_string: %s", o_query_string) @@ -206,17 +198,17 @@ def query_string_for_transaction_insert(self, table_name): The Sqlite3 query string used to insert a new row into the database. """ - o_query_string = f'INSERT INTO {table_name} ' + o_query_string = f"INSERT INTO {table_name} " - o_query_string += '(' + o_query_string += "(" for index, column in enumerate(self.DOI_DB_SCHEMA): - o_query_string += f'{column}' + o_query_string += f"{column}" if index < (self.EXPECTED_NUM_COLS - 1): - o_query_string += ',' + o_query_string += "," - o_query_string += ') ' + o_query_string += ") " o_query_string += f'VALUES ({",".join(["?"] * self.EXPECTED_NUM_COLS)})' @@ -244,11 +236,11 @@ def query_string_for_is_latest_update(self, table_name, primary_key_column): """ # Note that we set column "is_latest" to 0 to signify that all previous # rows are now not the latest. - o_query_string = f'UPDATE {table_name} ' - o_query_string += 'SET ' - o_query_string += 'is_latest = 0 ' - o_query_string += f'WHERE {primary_key_column} = ?' - o_query_string += ';' # Don't forget the last semi-colon for SQL to work. + o_query_string = f"UPDATE {table_name} " + o_query_string += "SET " + o_query_string += "is_latest = 0 " + o_query_string += f"WHERE {primary_key_column} = ?" + o_query_string += ";" # Don't forget the last semi-colon for SQL to work. logger.debug("UPDATE o_query_string: %s", o_query_string) @@ -264,13 +256,20 @@ def create_table(self, table_name): logger.info("Table created successfully") - def write_doi_info_to_database(self, identifier, transaction_key, doi=None, - date_added=datetime.now(), - date_updated=datetime.now(), - status=DoiStatus.Unknown, title='', - product_type=ProductType.Collection, - product_type_specific='', - submitter='', discipline_node=''): + def write_doi_info_to_database( + self, + identifier, + transaction_key, + doi=None, + date_added=datetime.now(), + date_updated=datetime.now(), + status=DoiStatus.Unknown, + title="", + product_type=ProductType.Collection, + product_type_specific="", + submitter="", + discipline_node="", + ): """ Write a new row to the Sqlite3 transaction database with the provided DOI entry information. @@ -321,32 +320,31 @@ def write_doi_info_to_database(self, identifier, transaction_key, doi=None, # Map the inputs to the appropriate column names. By doing so, we # can ignore database column ordering for now. data = { - 'identifier': identifier, - 'status': status, - 'date_added': date_added, - 'date_updated': date_updated, - 'submitter': submitter, - 'title': title, - 'type': product_type, - 'subtype': product_type_specific, - 'node_id': discipline_node, - 'doi': doi, - 'transaction_key': transaction_key, - 'is_latest': True + "identifier": identifier, + "status": status, + "date_added": date_added, + "date_updated": date_updated, + "submitter": submitter, + "title": title, + "type": product_type, + "subtype": product_type_specific, + "node_id": discipline_node, + "doi": doi, + "transaction_key": transaction_key, + "is_latest": True, } try: # Create and execute the query to unset the is_latest field for all # records with the same identifier field. query_string = self.query_string_for_is_latest_update( - self.m_default_table_name, primary_key_column='identifier' + self.m_default_table_name, primary_key_column="identifier" ) self.m_my_conn.execute(query_string, (identifier,)) self.m_my_conn.commit() except sqlite3.Error as err: - msg = (f"Failed to update is_latest field for identifier {identifier}, " - f"reason: {err}") + msg = f"Failed to update is_latest field for identifier {identifier}, " f"reason: {err}" logger.error(msg) raise RuntimeError(msg) @@ -361,8 +359,7 @@ def write_doi_info_to_database(self, identifier, transaction_key, doi=None, self.m_my_conn.execute(query_string, data_tuple) self.m_my_conn.commit() except sqlite3.Error as err: - msg = (f"Failed to commit transaction for identifier {identifier}, " - f"reason: {err}") + msg = f"Failed to commit transaction for identifier {identifier}, " f"reason: {err}" logger.error(msg) raise RuntimeError(msg) @@ -374,15 +371,16 @@ def _normalize_rows(self, columns, rows): for row in rows: # Convert the add/update times from Unix epoch back to datetime, # accounting for the expected (PST) timezone - for time_column in ('date_added', 'date_updated'): + for time_column in ("date_added", "date_updated"): time_val = row[columns.index(time_column)] - time_val = (datetime.fromtimestamp(time_val, tz=timezone.utc) - .replace(tzinfo=timezone(timedelta(hours=--8.0)))) + time_val = datetime.fromtimestamp(time_val, tz=timezone.utc).replace( + tzinfo=timezone(timedelta(hours=--8.0)) + ) row[columns.index(time_column)] = time_val # Convert status/product type back to Enums - row[columns.index('status')] = DoiStatus(row[columns.index('status')].lower()) - row[columns.index('type')] = ProductType(row[columns.index('type')].capitalize()) + row[columns.index("status")] = DoiStatus(row[columns.index("status")].lower()) + row[columns.index("type")] = ProductType(row[columns.index("type")].capitalize()) return rows @@ -393,14 +391,14 @@ def select_rows(self, query_criterias, table_name=None): self.m_my_conn = self.get_connection(table_name) - query_string = f'SELECT * FROM {table_name}' + query_string = f"SELECT * FROM {table_name}" criterias_str, criteria_dict = DOIDataBase.parse_criteria(query_criterias) if len(query_criterias) > 0: - query_string += f' WHERE {criterias_str}' + query_string += f" WHERE {criterias_str}" - query_string += '; ' + query_string += "; " logger.debug("SELECT query_string: %s", query_string) @@ -413,7 +411,7 @@ def select_rows(self, query_criterias, table_name=None): rows = self._normalize_rows(columns, rows) - logger.debug('Query returned %d result(s)', len(rows)) + logger.debug("Query returned %d result(s)", len(rows)) return columns, rows @@ -426,10 +424,9 @@ def select_latest_rows(self, query_criterias, table_name=None): criterias_str, criteria_dict = DOIDataBase.parse_criteria(query_criterias) - query_string = (f'SELECT * from {table_name} ' - f'WHERE is_latest=1 {criterias_str} ORDER BY date_updated') + query_string = f"SELECT * from {table_name} " f"WHERE is_latest=1 {criterias_str} ORDER BY date_updated" - logger.debug('SELECT query_string: %s', query_string) + logger.debug("SELECT query_string: %s", query_string) cursor = self.m_my_conn.cursor() cursor.execute(query_string, criteria_dict) @@ -440,7 +437,7 @@ def select_latest_rows(self, query_criterias, table_name=None): rows = self._normalize_rows(columns, rows) - logger.debug('Query returned %d result(s)', len(rows)) + logger.debug("Query returned %d result(s)", len(rows)) return columns, rows @@ -451,7 +448,7 @@ def select_all_rows(self, table_name=None): self.m_my_conn = self.get_connection(table_name) - query_string = f'SELECT * FROM {table_name};' + query_string = f"SELECT * FROM {table_name};" logger.debug("SELECT query_string %s", query_string) @@ -464,7 +461,7 @@ def select_all_rows(self, table_name=None): rows = self._normalize_rows(columns, rows) - logger.debug('Query returned %d result(s)', len(rows)) + logger.debug("Query returned %d result(s)", len(rows)) return columns, rows @@ -478,7 +475,7 @@ def update_rows(self, query_criterias, update_list, table_name=None): self.m_my_conn = self.get_connection(table_name) - query_string = f'UPDATE {table_name} SET ' + query_string = f"UPDATE {table_name} SET " for ii in range(len(update_list)): # Build the SET column_1 = new_value_1, @@ -487,18 +484,18 @@ def update_rows(self, query_criterias, update_list, table_name=None): if ii == 0: query_string += update_list[ii] else: - query_string += ',' + update_list[ii] + query_string += "," + update_list[ii] # Add any query_criterias if len(query_criterias) > 0: - query_string += ' WHERE ' + query_string += " WHERE " # Build the WHERE clause for ii in range(len(query_criterias)): if ii == 0: query_string += query_criterias[ii] else: - query_string += f' AND {query_criterias[ii]} ' + query_string += f" AND {query_criterias[ii]} " logger.debug("UPDATE query_string: %s", query_string) @@ -536,31 +533,31 @@ def _form_query_with_wildcards(column_name, search_tokens): """ # Partition the tokens containing wildcards from the fully specified ones - wildcard_tokens = list(filter(lambda token: '*' in token, search_tokens)) + wildcard_tokens = list(filter(lambda token: "*" in token, search_tokens)) full_tokens = list(set(search_tokens) - set(wildcard_tokens)) # Clean up the column name provided so it can be used as a suitable # named parameter placeholder token - filter_chars = [' ', '\'', ':', '|'] + filter_chars = [" ", "'", ":", "|"] named_param_id = column_name for filter_char in filter_chars: - named_param_id = named_param_id.replace(filter_char, '') + named_param_id = named_param_id.replace(filter_char, "") # Set up the named parameters for the IN portion of the WHERE used # to find fully specified tokens - named_parameters = ','.join([f':{named_param_id}_{i}' - for i in range(len(full_tokens))]) - named_parameter_values = {f'{named_param_id}_{i}': full_tokens[i] - for i in range(len(full_tokens))} + named_parameters = ",".join([f":{named_param_id}_{i}" for i in range(len(full_tokens))]) + named_parameter_values = {f"{named_param_id}_{i}": full_tokens[i] for i in range(len(full_tokens))} # Set up the named parameters for the GLOB portion of the WHERE used # find tokens containing wildcards - glob_parameters = ' OR '.join([f'{column_name} GLOB :{named_param_id}_glob_{i}' - for i in range(len(wildcard_tokens))]) + glob_parameters = " OR ".join( + [f"{column_name} GLOB :{named_param_id}_glob_{i}" for i in range(len(wildcard_tokens))] + ) - named_parameter_values.update({f'{named_param_id}_glob_{i}': wildcard_tokens[i] - for i in range(len(wildcard_tokens))}) + named_parameter_values.update( + {f"{named_param_id}_glob_{i}": wildcard_tokens[i] for i in range(len(wildcard_tokens))} + ) # Build the portion of the WHERE clause combining the necessary # parameters needed to search for all the tokens we were provided @@ -570,10 +567,10 @@ def _form_query_with_wildcards(column_name, search_tokens): where_subclause += f"{column_name} IN ({named_parameters}) " if full_tokens and wildcard_tokens: - where_subclause += ' OR ' + where_subclause += " OR " if wildcard_tokens: - where_subclause += f'{glob_parameters}' + where_subclause += f"{glob_parameters}" where_subclause += ")" @@ -583,53 +580,51 @@ def _form_query_with_wildcards(column_name, search_tokens): @staticmethod def _get_simple_in_criteria(v, column): - named_parameters = ','.join([':' + column + '_' + str(i) for i in range(len(v))]) - named_parameter_values = {column + '_' + str(i): v[i].lower() for i in range(len(v))} - return f' AND lower({column}) IN ({named_parameters})', named_parameter_values + named_parameters = ",".join([":" + column + "_" + str(i) for i in range(len(v))]) + named_parameter_values = {column + "_" + str(i): v[i].lower() for i in range(len(v))} + return f" AND lower({column}) IN ({named_parameters})", named_parameter_values @staticmethod def _get_query_criteria_title(v): - return DOIDataBase._get_simple_in_criteria(v, 'title') + return DOIDataBase._get_simple_in_criteria(v, "title") @staticmethod def _get_query_criteria_doi(v): - return DOIDataBase._get_simple_in_criteria(v, 'doi') + return DOIDataBase._get_simple_in_criteria(v, "doi") @staticmethod def _get_query_criteria_ids(v): - return DOIDataBase._form_query_with_wildcards('identifier', v) + return DOIDataBase._form_query_with_wildcards("identifier", v) @staticmethod def _get_query_criteria_submitter(v): - return DOIDataBase._get_simple_in_criteria(v, 'submitter') + return DOIDataBase._get_simple_in_criteria(v, "submitter") @staticmethod def _get_query_criteria_node(v): - return DOIDataBase._get_simple_in_criteria(v, 'node_id') + return DOIDataBase._get_simple_in_criteria(v, "node_id") @staticmethod def _get_query_criteria_status(v): - return DOIDataBase._get_simple_in_criteria(v, 'status') + return DOIDataBase._get_simple_in_criteria(v, "status") @staticmethod def _get_query_criteria_start_update(v): - return (' AND date_updated >= :start_update', - {'start_update': v.replace(tzinfo=timezone.utc).timestamp()}) + return (" AND date_updated >= :start_update", {"start_update": v.replace(tzinfo=timezone.utc).timestamp()}) @staticmethod def _get_query_criteria_end_update(v): - return (' AND date_updated <= :end_update', - {'end_update': v.replace(tzinfo=timezone.utc).timestamp()}) + return (" AND date_updated <= :end_update", {"end_update": v.replace(tzinfo=timezone.utc).timestamp()}) @staticmethod def parse_criteria(query_criterias): - criterias_str = '' + criterias_str = "" criteria_dict = {} for k, v in query_criterias.items(): logger.debug("Calling get_query_criteria_%s with value %s", k, v) - criteria_str, dict_entry = getattr(DOIDataBase, '_get_query_criteria_' + k)(v) + criteria_str, dict_entry = getattr(DOIDataBase, "_get_query_criteria_" + k)(v) logger.debug("criteria_str: %s", criteria_str) logger.debug("dict_entry: %s", dict_entry) diff --git a/src/pds_doi_service/core/db/test/__init__.py b/src/pds_doi_service/core/db/test/__init__.py index c3ea2bb2..df26172b 100644 --- a/src/pds_doi_service/core/db/test/__init__.py +++ b/src/pds_doi_service/core/db/test/__init__.py @@ -1,11 +1,9 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for core database -''' - - +""" import unittest + from . import doi_database_test diff --git a/src/pds_doi_service/core/db/test/doi_database_test.py b/src/pds_doi_service/core/db/test/doi_database_test.py index 86cd1043..6f9f4a5b 100644 --- a/src/pds_doi_service/core/db/test/doi_database_test.py +++ b/src/pds_doi_service/core/db/test/doi_database_test.py @@ -1,16 +1,16 @@ #!/usr/bin/env python - import datetime -from datetime import timedelta, timezone import os -from os.path import exists import unittest - -from pkg_resources import resource_filename +from datetime import timedelta +from datetime import timezone +from os.path import exists from pds_doi_service.core.db.doi_database import DOIDataBase -from pds_doi_service.core.entities.doi import DoiStatus, ProductType +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.util.general_util import get_logger +from pkg_resources import resource_filename logger = get_logger(__name__) @@ -19,7 +19,7 @@ class DOIDatabaseTest(unittest.TestCase): """Unit tests for the doi_database.py module""" def setUp(self): - self._db_name = resource_filename(__name__, 'doi_temp.db') + self._db_name = resource_filename(__name__, "doi_temp.db") # Delete temporary db if it already exists, this can occur when tests # are terminated before completion (during debugging for example) @@ -35,158 +35,186 @@ def tearDown(self): def test_select_latest_rows(self): """Test selecting of latest rows from the transaction database""" # Set up a sample db entry - identifier = 'urn:nasa:pds:lab_shocked_feldspars::1.0' - transaction_key = 'img/2020-06-15T18:42:45.653317' - doi = '10.17189/21729' + identifier = "urn:nasa:pds:lab_shocked_feldspars::1.0" + transaction_key = "img/2020-06-15T18:42:45.653317" + doi = "10.17189/21729" date_added = datetime.datetime.now() date_updated = datetime.datetime.now() status = DoiStatus.Unknown - title = 'Laboratory Shocked Feldspars Bundle' + title = "Laboratory Shocked Feldspars Bundle" product_type = ProductType.Collection - product_type_specific = 'PDS4 Collection' - submitter = 'img-submitter@jpl.nasa.gov' - discipline_node = 'img' + product_type_specific = "PDS4 Collection" + submitter = "img-submitter@jpl.nasa.gov" + discipline_node = "img" # Insert a row in the 'doi' table self._doi_database.write_doi_info_to_database( - identifier, transaction_key, doi, date_added, date_updated, status, - title, product_type, product_type_specific, submitter, discipline_node + identifier, + transaction_key, + doi, + date_added, + date_updated, + status, + title, + product_type, + product_type_specific, + submitter, + discipline_node, ) # Select the row we just added # The type of o_query_result should be JSON and a list of 1 - o_query_result = self._doi_database.select_latest_rows( - query_criterias={'doi': [doi]} - ) + o_query_result = self._doi_database.select_latest_rows(query_criterias={"doi": [doi]}) # Reformat results to a dictionary to test with query_result = dict(zip(o_query_result[0], o_query_result[1][0])) # Ensure we got back everything we just put in - self.assertEqual(query_result['status'], status) + self.assertEqual(query_result["status"], status) self.assertEqual( - int(query_result['date_added'].timestamp()), - int(date_added.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()) + int(query_result["date_added"].timestamp()), + int(date_added.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()), ) self.assertEqual( - int(query_result['date_updated'].timestamp()), - int(date_updated.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()) + int(query_result["date_updated"].timestamp()), + int(date_updated.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()), ) - self.assertEqual(query_result['submitter'], submitter) - self.assertEqual(query_result['title'], title) - self.assertEqual(query_result['type'], product_type) - self.assertEqual(query_result['subtype'], product_type_specific) - self.assertEqual(query_result['node_id'], discipline_node) - self.assertEqual(query_result['identifier'], identifier) - self.assertEqual(query_result['doi'], doi) - self.assertEqual(query_result['transaction_key'], transaction_key) + self.assertEqual(query_result["submitter"], submitter) + self.assertEqual(query_result["title"], title) + self.assertEqual(query_result["type"], product_type) + self.assertEqual(query_result["subtype"], product_type_specific) + self.assertEqual(query_result["node_id"], discipline_node) + self.assertEqual(query_result["identifier"], identifier) + self.assertEqual(query_result["doi"], doi) + self.assertEqual(query_result["transaction_key"], transaction_key) - self.assertTrue(query_result['is_latest']) + self.assertTrue(query_result["is_latest"]) # Update some fields and write a new "latest" entry status = DoiStatus.Draft - submitter = 'eng-submitter@jpl.nasa.gov' - discipline_node = 'eng' + submitter = "eng-submitter@jpl.nasa.gov" + discipline_node = "eng" self._doi_database.write_doi_info_to_database( - identifier, transaction_key, doi, date_added, date_updated, status, - title, product_type, product_type_specific, submitter, discipline_node + identifier, + transaction_key, + doi, + date_added, + date_updated, + status, + title, + product_type, + product_type_specific, + submitter, + discipline_node, ) # Query again and ensure we only get latest back - o_query_result = self._doi_database.select_latest_rows( - query_criterias={'doi': [doi]} - ) + o_query_result = self._doi_database.select_latest_rows(query_criterias={"doi": [doi]}) # Should only get the one row back self.assertEqual(len(o_query_result[-1]), 1) query_result = dict(zip(o_query_result[0], o_query_result[-1][0])) - self.assertEqual(query_result['status'], status) - self.assertEqual(query_result['submitter'], submitter) - self.assertEqual(query_result['node_id'], discipline_node) + self.assertEqual(query_result["status"], status) + self.assertEqual(query_result["submitter"], submitter) + self.assertEqual(query_result["node_id"], discipline_node) - self.assertTrue(query_result['is_latest']) + self.assertTrue(query_result["is_latest"]) self._doi_database.close_database() def test_select_latest_rows_lid_only(self): """Test corner case where we select and update rows that only specify a LID""" # Set up a sample db entry - identifier = 'urn:nasa:pds:insight_cameras' - transaction_key = 'img/2021-05-10T00:00:00.000000' - doi = '10.17189/22000' + identifier = "urn:nasa:pds:insight_cameras" + transaction_key = "img/2021-05-10T00:00:00.000000" + doi = "10.17189/22000" date_added = datetime.datetime.now() date_updated = datetime.datetime.now() status = DoiStatus.Unknown - title = 'Insight Cameras Bundle' + title = "Insight Cameras Bundle" product_type = ProductType.Collection - product_type_specific = 'PDS4 Collection' - submitter = 'eng-submitter@jpl.nasa.gov' - discipline_node = 'eng' + product_type_specific = "PDS4 Collection" + submitter = "eng-submitter@jpl.nasa.gov" + discipline_node = "eng" # Insert a row in the 'doi' table self._doi_database.write_doi_info_to_database( - identifier, transaction_key, doi, date_added, date_updated, status, - title, product_type, product_type_specific, submitter, discipline_node + identifier, + transaction_key, + doi, + date_added, + date_updated, + status, + title, + product_type, + product_type_specific, + submitter, + discipline_node, ) # Select the row we just added # The type of o_query_result should be JSON and a list of 1 - o_query_result = self._doi_database.select_latest_rows( - query_criterias={'ids': [identifier]} - ) + o_query_result = self._doi_database.select_latest_rows(query_criterias={"ids": [identifier]}) # Reformat results to a dictionary to test with query_result = dict(zip(o_query_result[0], o_query_result[1][0])) # Ensure we got back everything we just put in - self.assertEqual(query_result['status'], status) + self.assertEqual(query_result["status"], status) self.assertEqual( - int(query_result['date_added'].timestamp()), - int(date_added.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()) + int(query_result["date_added"].timestamp()), + int(date_added.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()), ) self.assertEqual( - int(query_result['date_updated'].timestamp()), - int(date_updated.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()) + int(query_result["date_updated"].timestamp()), + int(date_updated.replace(tzinfo=timezone(timedelta(hours=--8.0))).timestamp()), ) - self.assertEqual(query_result['submitter'], submitter) - self.assertEqual(query_result['title'], title) - self.assertEqual(query_result['type'], product_type) - self.assertEqual(query_result['subtype'], product_type_specific) - self.assertEqual(query_result['node_id'], discipline_node) - self.assertEqual(query_result['identifier'], identifier) - self.assertEqual(query_result['doi'], doi) - self.assertEqual(query_result['transaction_key'], transaction_key) + self.assertEqual(query_result["submitter"], submitter) + self.assertEqual(query_result["title"], title) + self.assertEqual(query_result["type"], product_type) + self.assertEqual(query_result["subtype"], product_type_specific) + self.assertEqual(query_result["node_id"], discipline_node) + self.assertEqual(query_result["identifier"], identifier) + self.assertEqual(query_result["doi"], doi) + self.assertEqual(query_result["transaction_key"], transaction_key) - self.assertTrue(query_result['is_latest']) + self.assertTrue(query_result["is_latest"]) # Update some fields and write a new "latest" entry status = DoiStatus.Pending - submitter = 'img-submitter@jpl.nasa.gov' - discipline_node = 'img' + submitter = "img-submitter@jpl.nasa.gov" + discipline_node = "img" self._doi_database.write_doi_info_to_database( - identifier, transaction_key, doi, date_added, date_updated, status, - title, product_type, product_type_specific, submitter, discipline_node + identifier, + transaction_key, + doi, + date_added, + date_updated, + status, + title, + product_type, + product_type_specific, + submitter, + discipline_node, ) # Query again and ensure we only get latest back - o_query_result = self._doi_database.select_latest_rows( - query_criterias={'ids': [identifier]} - ) + o_query_result = self._doi_database.select_latest_rows(query_criterias={"ids": [identifier]}) # Should only get the one row back self.assertEqual(len(o_query_result[-1]), 1) query_result = dict(zip(o_query_result[0], o_query_result[-1][0])) - self.assertEqual(query_result['status'], status) - self.assertEqual(query_result['submitter'], submitter) - self.assertEqual(query_result['node_id'], discipline_node) + self.assertEqual(query_result["status"], status) + self.assertEqual(query_result["submitter"], submitter) + self.assertEqual(query_result["node_id"], discipline_node) - self.assertTrue(query_result['is_latest']) + self.assertTrue(query_result["is_latest"]) self._doi_database.close_database() @@ -199,29 +227,38 @@ def test_query_by_wildcard(self): num_rows = 6 for _id in range(1, 1 + num_rows): - lid = 'urn:nasa:pds:lab_shocked_feldspars' - vid = f'{_id}.0' - identifier = lid + '::' + vid - transaction_key = f'img/{_id}/2020-06-15T18:42:45.653317' - doi = f'10.17189/2000{_id}' + lid = "urn:nasa:pds:lab_shocked_feldspars" + vid = f"{_id}.0" + identifier = lid + "::" + vid + transaction_key = f"img/{_id}/2020-06-15T18:42:45.653317" + doi = f"10.17189/2000{_id}" date_added = datetime.datetime.now() date_updated = datetime.datetime.now() status = DoiStatus.Draft - title = f'Laboratory Shocked Feldspars Bundle {_id}' + title = f"Laboratory Shocked Feldspars Bundle {_id}" product_type = ProductType.Collection - product_type_specific = 'PDS4 Collection' - submitter = 'img-submitter@jpl.nasa.gov' - discipline_node = 'img' + product_type_specific = "PDS4 Collection" + submitter = "img-submitter@jpl.nasa.gov" + discipline_node = "img" self._doi_database.write_doi_info_to_database( - identifier, transaction_key, doi, date_added, date_updated, status, - title, product_type, product_type_specific, submitter, discipline_node + identifier, + transaction_key, + doi, + date_added, + date_updated, + status, + title, + product_type, + product_type_specific, + submitter, + discipline_node, ) # Use a wildcard with lidvid column to select everything we just # inserted o_query_result = self._doi_database.select_latest_rows( - query_criterias={'ids': ['urn:nasa:pds:lab_shocked_feldspars::*']} + query_criterias={"ids": ["urn:nasa:pds:lab_shocked_feldspars::*"]} ) # Should get all rows back @@ -229,19 +266,20 @@ def test_query_by_wildcard(self): # Try again using just the lid o_query_result = self._doi_database.select_latest_rows( - query_criterias={'ids': ['urn:nasa:*:lab_shocked_feldspars::1.0']} + query_criterias={"ids": ["urn:nasa:*:lab_shocked_feldspars::1.0"]} ) self.assertEqual(len(o_query_result[-1]), 1) # Test with a combination of wildcards and full tokens o_query_result = self._doi_database.select_latest_rows( - query_criterias={'ids': ['urn:nasa:pds:lab_shocked_feldspars::1.0', - 'urn:nasa:pds:lab_shocked_feldspars::2.*']} + query_criterias={ + "ids": ["urn:nasa:pds:lab_shocked_feldspars::1.0", "urn:nasa:pds:lab_shocked_feldspars::2.*"] + } ) self.assertEqual(len(o_query_result[-1]), 2) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/entities/doi.py b/src/pds_doi_service/core/entities/doi.py index 6b450970..769adf95 100644 --- a/src/pds_doi_service/core/entities/doi.py +++ b/src/pds_doi_service/core/entities/doi.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ====== doi.py @@ -12,20 +11,22 @@ Contains the dataclass and enumeration definitions for Doi objects. """ - -from dataclasses import dataclass, field +from dataclasses import dataclass +from dataclasses import field from datetime import datetime -from enum import Enum, unique +from enum import Enum +from enum import unique @unique class ProductType(str, Enum): """Enumerates the types of products that can be assigned a DOI.""" - Collection = 'Collection' - Bundle = 'Bundle' - Text = 'Text' - Dataset = 'Dataset' - Other = 'Other' + + Collection = "Collection" + Bundle = "Bundle" + Text = "Text" + Dataset = "Dataset" + Other = "Other" @unique @@ -63,16 +64,17 @@ class DoiStatus(str, Enum): The submitted DOI has been deactivated (deleted). """ - Error = 'error' - Unknown = 'unknown' - Reserved_not_submitted = 'reserved_not_submitted' - Reserved = 'reserved' - Draft = 'draft' - Review = 'review' - Pending = 'pending' - Registered = 'registered' - Findable = 'findable' - Deactivated = 'deactivated' + + Error = "error" + Unknown = "unknown" + Reserved_not_submitted = "reserved_not_submitted" + Reserved = "reserved" + Draft = "draft" + Review = "review" + Pending = "pending" + Registered = "registered" + Findable = "findable" + Deactivated = "deactivated" @unique @@ -90,14 +92,16 @@ class DoiEvent(str, Enum): Moves a DOI from findable back to registered """ - Publish = 'publish' - Register = 'register' - Hide = 'hide' + + Publish = "publish" + Register = "register" + Hide = "hide" @dataclass class Doi: """The dataclass definition for a Doi object.""" + title: str publication_date: datetime product_type: ProductType diff --git a/src/pds_doi_service/core/input/exceptions.py b/src/pds_doi_service/core/input/exceptions.py index df739fb7..b1185faf 100644 --- a/src/pds_doi_service/core/input/exceptions.py +++ b/src/pds_doi_service/core/input/exceptions.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============= exceptions.py @@ -12,24 +11,26 @@ Contains exception classes and functions for collecting and managing exceptions. """ - from pds_doi_service.core.util.general_util import get_logger -logger = get_logger('pds_doi_service.core.input.exceptions') +logger = get_logger("pds_doi_service.core.input.exceptions") class InputFormatException(Exception): """Raised when an input file is not formatted as expected.""" + pass class UnknownNodeException(Exception): """Raised when an unknown PDS Node identifier is provided.""" + pass class UnknownIdentifierException(Exception): """Raised when no corresponding DOI entry can be found for a given PDS ID.""" + pass @@ -39,11 +40,13 @@ class InvalidIdentifierException(Exception): class NoTransactionHistoryForIdentifierException(Exception): """Raised when no transaction database entry can be found for a given PDS ID.""" + pass class DuplicatedTitleDOIException(Exception): """Raised when a DOI title has already been used with another ID.""" + pass @@ -53,6 +56,7 @@ class InvalidRecordException(Exception): class IllegalDOIActionException(Exception): """Raised when attempting to create or modify a DOI for an existing ID.""" + pass @@ -61,16 +65,19 @@ class UnexpectedDOIActionException(Exception): Raised when a DOI has an unexpected status, or a requested action circumvents the expected DOI workflow. """ + pass class TitleDoesNotMatchProductTypeException(Exception): """Raised when a DOI's title does not contain the product type.""" + pass class CriticalDOIException(Exception): """Raised for any exceptions that are not handled with another class.""" + pass @@ -79,11 +86,13 @@ class WarningDOIException(Exception): Used to roll up multiple exceptions or warnings encountered while processing multiple DOI entries. """ + pass class SiteURLNotExistException(Exception): """Raised when a DOI's site URL cannot be reached.""" + pass @@ -91,9 +100,7 @@ class WebRequestException(Exception): """Raised when a request to the DOI endpoint service fails.""" -def collect_exception_classes_and_messages(single_exception, - io_exception_classes, - io_exception_messages): +def collect_exception_classes_and_messages(single_exception, io_exception_classes, io_exception_messages): """ Given a single exception, collect the exception class name and message. The variables io_exception_classes and io_exception_messages are both @@ -118,8 +125,7 @@ def collect_exception_classes_and_messages(single_exception, """ # ex: SiteURNotExistException actual_class_name = type(single_exception).__name__ - logger.debug("actual_class_name,type(actual_class_name) " - f"{actual_class_name},{type(actual_class_name)}") + logger.debug("actual_class_name,type(actual_class_name) " f"{actual_class_name},{type(actual_class_name)}") io_exception_classes.append(actual_class_name) @@ -151,18 +157,14 @@ def raise_or_warn_exceptions(exception_classes, exception_messages, log=False): and messages. """ - message_to_raise = '' + message_to_raise = "" for ii in range(len(exception_classes)): if ii == 0: - message_to_raise = (message_to_raise - + exception_classes[ii] - + ' : ' + exception_messages[ii]) + message_to_raise = message_to_raise + exception_classes[ii] + " : " + exception_messages[ii] else: # Add a comma after every message. - message_to_raise = (message_to_raise - + ', ' + exception_classes[ii] - + ' : ' + exception_messages[ii]) + message_to_raise = message_to_raise + ", " + exception_classes[ii] + " : " + exception_messages[ii] if log: logger.warning(message_to_raise) diff --git a/src/pds_doi_service/core/input/input_util.py b/src/pds_doi_service/core/input/input_util.py index 86afa5b1..278076da 100644 --- a/src/pds_doi_service/core/input/input_util.py +++ b/src/pds_doi_service/core/input/input_util.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============= input_util.py @@ -12,28 +11,28 @@ Contains classes for working with input label files, be they local or remote. """ - import os -import urllib.parse import tempfile +import urllib.parse from datetime import datetime from os.path import basename -from xmlschema import XMLSchemaValidationError - import pandas as pd import requests from lxml import etree - -from pds_doi_service.core.entities.doi import Doi, DoiStatus, ProductType +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.input.exceptions import InputFormatException from pds_doi_service.core.input.pds4_util import DOIPDS4LabelUtil from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON from pds_doi_service.core.outputs.osti.osti_validator import DOIOstiValidator from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiXmlWebParser -from pds_doi_service.core.outputs.service import DOIServiceFactory, SERVICE_TYPE_DATACITE +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger +from xmlschema import XMLSchemaValidationError # Get the common logger logger = get_logger(__name__) @@ -44,15 +43,21 @@ class DOIInputUtil: EXPECTED_NUM_COLUMNS = 7 """Expected number of columns in an input CSV file.""" - MANDATORY_COLUMNS = ['status', 'title', 'publication_date', - 'product_type_specific', 'author_last_name', - 'author_first_name', 'related_resource'] + MANDATORY_COLUMNS = [ + "status", + "title", + "publication_date", + "product_type_specific", + "author_last_name", + "author_first_name", + "related_resource", + ] """The names of the expected columns within a CSV file.""" EXPECTED_PUBLICATION_DATE_LEN = 10 """Expected minimum length of a parsed publication date.""" - DEFAULT_VALID_EXTENSIONS = ['.xml', '.csv', '.xlsx', '.xls', '.json'] + DEFAULT_VALID_EXTENSIONS = [".xml", ".csv", ".xlsx", ".xls", ".json"] """The default list of valid input file extensions this module can read.""" def __init__(self, valid_extensions=None): @@ -74,9 +79,7 @@ def __init__(self, valid_extensions=None): """ self._config = DOIConfigUtil().get_config() - self._label_util = DOIPDS4LabelUtil( - landing_page_template=self._config.get('LANDING_PAGES', 'url') - ) + self._label_util = DOIPDS4LabelUtil(landing_page_template=self._config.get("LANDING_PAGES", "url")) self._valid_extensions = valid_extensions or self.DEFAULT_VALID_EXTENSIONS if not isinstance(self._valid_extensions, (list, tuple, set)): @@ -85,17 +88,15 @@ def __init__(self, valid_extensions=None): # Set up the mapping of supported extensions to the corresponding read # function pointers self._parser_map = { - '.xml': self.parse_xml_file, - '.xls': self.parse_xls_file, - '.xlsx': self.parse_xls_file, - '.csv': self.parse_csv_file, - '.json': self.parse_json_file + ".xml": self.parse_xml_file, + ".xls": self.parse_xls_file, + ".xlsx": self.parse_xls_file, + ".csv": self.parse_csv_file, + ".json": self.parse_json_file, } - if not all([extension in self._parser_map - for extension in self._valid_extensions]): - raise ValueError('One or more the provided extensions are not ' - 'supported by the DOIInputUtil class.') + if not all([extension in self._parser_map for extension in self._valid_extensions]): + raise ValueError("One or more the provided extensions are not " "supported by the DOIInputUtil class.") def parse_xml_file(self, xml_path): """ @@ -122,27 +123,22 @@ def parse_xml_file(self, xml_path): dois = [] # First read the contents of the file - with open(xml_path, 'r') as infile: + with open(xml_path, "r") as infile: xml_contents = infile.read() xml_tree = etree.fromstring(xml_contents.encode()) # Check if we were handed a PSD4 label if self._label_util.is_pds4_label(xml_tree): - logger.info('Parsing xml file %s as a PSD4 label', - basename(xml_path)) + logger.info("Parsing xml file %s as a PSD4 label", basename(xml_path)) try: dois.append(self._label_util.get_doi_fields_from_pds4(xml_tree)) except Exception as err: - raise InputFormatException( - 'Could not parse the provided xml file as a PDS4 label.\n' - f'Reason: {err}' - ) + raise InputFormatException("Could not parse the provided xml file as a PDS4 label.\n" f"Reason: {err}") # Otherwise, assume OSTI format else: - logger.info('Parsing xml file %s as an OSTI label', - basename(xml_path)) + logger.info("Parsing xml file %s as an OSTI label", basename(xml_path)) try: DOIOstiValidator()._validate_against_xsd(xml_tree) @@ -150,8 +146,7 @@ def parse_xml_file(self, xml_path): dois, _ = DOIOstiXmlWebParser.parse_dois_from_label(xml_contents) except XMLSchemaValidationError as err: raise InputFormatException( - 'Could not parse the provided xml file as an OSTI label.\n' - f'Reason: {err.reason}' + "Could not parse the provided xml file as an OSTI label.\n" f"Reason: {err.reason}" ) return dois @@ -178,19 +173,20 @@ def parse_xls_file(self, xls_path): of columns. """ - logger.info('Parsing xls file %s', basename(xls_path)) + logger.info("Parsing xls file %s", basename(xls_path)) - xl_wb = pd.ExcelFile(xls_path, engine='openpyxl') + xl_wb = pd.ExcelFile(xls_path, engine="openpyxl") # We only want the first sheet. actual_sheet_name = xl_wb.sheet_names[0] xl_sheet = pd.read_excel( - xls_path, actual_sheet_name, + xls_path, + actual_sheet_name, # Parse 3rd column (1-indexed) as a pd.Timestamp, can't use # name of column since it hasn't been standardized yet parse_dates=[3], # Remove automatic replacement of empty columns with NaN - na_filter=False + na_filter=False, ) num_cols = len(xl_sheet.columns) @@ -203,15 +199,17 @@ def parse_xls_file(self, xls_path): # rename columns in a simpler way xl_sheet = xl_sheet.rename( columns={ - 'publication_date (yyyy-mm-dd)': 'publication_date', - 'product_type_specific\n(PDS4 Bundle | PDS4 Collection | PDS4 Document)': 'product_type_specific', - 'related_resource\nLIDVID': 'related_resource' + "publication_date (yyyy-mm-dd)": "publication_date", + "product_type_specific\n(PDS4 Bundle | PDS4 Collection | PDS4 Document)": "product_type_specific", + "related_resource\nLIDVID": "related_resource", } ) if num_cols < self.EXPECTED_NUM_COLUMNS: - msg = (f"Expected {self.EXPECTED_NUM_COLUMNS} columns in the " - f"provided XLS file, but only found {num_cols} column(s).") + msg = ( + f"Expected {self.EXPECTED_NUM_COLUMNS} columns in the " + f"provided XLS file, but only found {num_cols} column(s)." + ) logger.error(msg) raise InputFormatException(msg) @@ -242,16 +240,17 @@ def _parse_rows_to_dois(self, xl_sheet): for index, row in xl_sheet.iterrows(): logger.debug(f"row {row}") - doi = Doi(status=DoiStatus(row['status'].lower()), - title=row['title'], - publication_date=row['publication_date'], - product_type=self._parse_product_type(row['product_type_specific']), - product_type_specific=row['product_type_specific'], - related_identifier=row['related_resource'], - authors=[{'first_name': row['author_first_name'], - 'last_name': row['author_last_name']}], - date_record_added=timestamp, - date_record_updated=timestamp) + doi = Doi( + status=DoiStatus(row["status"].lower()), + title=row["title"], + publication_date=row["publication_date"], + product_type=self._parse_product_type(row["product_type_specific"]), + product_type_specific=row["product_type_specific"], + related_identifier=row["related_resource"], + authors=[{"first_name": row["author_first_name"], "last_name": row["author_last_name"]}], + date_record_added=timestamp, + date_record_updated=timestamp, + ) logger.debug("Parsed Doi: %r", doi.__dict__) dois.append(doi) @@ -280,11 +279,10 @@ def _parse_product_type(product_type_specific): try: product_type = ProductType(product_type_specific_suffix.capitalize()) - logger.debug('Parsed %s from %s', product_type, product_type_specific) + logger.debug("Parsed %s from %s", product_type, product_type_specific) except ValueError: product_type = ProductType.Collection - logger.debug('Could not parsed product type from %s, defaulting to %s', - product_type_specific, product_type) + logger.debug("Could not parsed product type from %s, defaulting to %s", product_type_specific, product_type) return product_type @@ -310,7 +308,7 @@ def parse_csv_file(self, csv_path): of columns. """ - logger.info('Parsing csv file %s', basename(csv_path)) + logger.info("Parsing csv file %s", basename(csv_path)) # Read the CSV file into memory csv_sheet = pd.read_csv( @@ -318,7 +316,7 @@ def parse_csv_file(self, csv_path): # Have pandas auto-parse publication_date column as a datetime parse_dates=["publication_date"], # Remove automatic replacement of empty columns with NaN - na_filter=False + na_filter=False, ) num_cols = len(csv_sheet.columns) @@ -330,8 +328,10 @@ def parse_csv_file(self, csv_path): logger.debug("data columns: %s", str(list(csv_sheet.columns))) if num_cols < self.EXPECTED_NUM_COLUMNS: - msg = (f"Expecting {self.EXPECTED_NUM_COLUMNS} columns in the provided " - f"CSV file, but only found {num_cols} column(s).") + msg = ( + f"Expecting {self.EXPECTED_NUM_COLUMNS} columns in the provided " + f"CSV file, but only found {num_cols} column(s)." + ) logger.error(msg) raise InputFormatException(msg) @@ -356,14 +356,14 @@ def parse_json_file(self, json_path): DOI objects parsed from the provided JSON file. """ - logger.info('Parsing json file %s', basename(json_path)) + logger.info("Parsing json file %s", basename(json_path)) dois = [] web_parser = DOIServiceFactory.get_web_parser_service() validator = DOIServiceFactory.get_validator_service() # First read the contents of the file - with open(json_path, 'r') as infile: + with open(json_path, "r") as infile: json_contents = infile.read() # Validate and parse the provide JSON label based on the service provider @@ -373,13 +373,11 @@ def parse_json_file(self, json_path): if DOIServiceFactory.get_service_type() == SERVICE_TYPE_DATACITE: validator.validate(json_contents) - dois, _ = web_parser.parse_dois_from_label( - json_contents, content_type=CONTENT_TYPE_JSON - ) + dois, _ = web_parser.parse_dois_from_label(json_contents, content_type=CONTENT_TYPE_JSON) except InputFormatException as err: - logger.warning('Unable to parse DOI objects from provided ' - 'json file "%s"\nReason: %s', - json_path, str(err)) + logger.warning( + "Unable to parse DOI objects from provided " 'json file "%s"\nReason: %s', json_path, str(err) + ) return dois @@ -404,7 +402,7 @@ def _read_from_path(self, path): dois = [] if os.path.isfile(path): - logger.info('Reading local file path %s', path) + logger.info("Reading local file path %s", path) extension = os.path.splitext(path)[-1] @@ -415,14 +413,14 @@ def _read_from_path(self, path): try: dois = read_function(path) except OSError as err: - msg = f'Error reading file {path}, reason: {str(err)}' + msg = f"Error reading file {path}, reason: {str(err)}" logger.error(msg) raise InputFormatException(msg) else: - logger.info('File %s has unsupported extension, ignoring', path) + logger.info("File %s has unsupported extension, ignoring", path) else: - logger.info('Reading files within directory %s', path) + logger.info("Reading files within directory %s", path) for sub_path in os.listdir(path): dois.extend(self._read_from_path(os.path.join(path, sub_path))) @@ -467,9 +465,7 @@ def _read_from_remote(self, input_url): try: response.raise_for_status() except requests.exceptions.HTTPError as http_err: - raise InputFormatException( - f'Could not read remote file {input_url}, reason: {str(http_err)}' - ) + raise InputFormatException(f"Could not read remote file {input_url}, reason: {str(http_err)}") with tempfile.NamedTemporaryFile(suffix=basename(parsed_url.path)) as temp_file: temp_file.write(response.content) @@ -505,15 +501,14 @@ def parse_dois_from_input_file(self, input_file): """ # See if we were handed a URL - if input_file.startswith('http'): + if input_file.startswith("http"): dois = self._read_from_remote(input_file) # Otherwise see if its a local file elif os.path.exists(input_file): dois = self._read_from_path(input_file) else: raise InputFormatException( - f"Error reading file {input_file}, path does not correspond to " - f"a remote URL or a local file path." + f"Error reading file {input_file}, path does not correspond to " f"a remote URL or a local file path." ) # Make sure we got back at least one Doi diff --git a/src/pds_doi_service/core/input/node_util.py b/src/pds_doi_service/core/input/node_util.py index 71c76764..515d6221 100644 --- a/src/pds_doi_service/core/input/node_util.py +++ b/src/pds_doi_service/core/input/node_util.py @@ -5,39 +5,43 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # -#------------------------------ - -from pds_doi_service.core.util.general_util import get_logger +# ------------------------------ from pds_doi_service.core.input.exceptions import UnknownNodeException +from pds_doi_service.core.util.general_util import get_logger # Get the common logger and set the level for this file. -logger = get_logger('pds_doi_core.input.node_util') +logger = get_logger("pds_doi_core.input.node_util") + class NodeUtil: # This class NodeUtil provide services to look up a short name for a long name of the node id. - m_node_id_dict = {'ATM': 'Atmospheres', - 'ENG': 'Engineering', - 'GEO': 'Geosciences', - 'IMG': 'Cartography and Imaging Sciences Discipline', - 'NAIF': 'Navigational and Ancillary Information Facility', - 'PPI': 'Planetary Plasma Interactions', - 'RMS': 'Ring-Moon Systems', - 'SBN': 'Small Bodies'} - - - def get_node_long_name(self,node_id): + m_node_id_dict = { + "ATM": "Atmospheres", + "ENG": "Engineering", + "GEO": "Geosciences", + "IMG": "Cartography and Imaging Sciences Discipline", + "NAIF": "Navigational and Ancillary Information Facility", + "PPI": "Planetary Plasma Interactions", + "RMS": "Ring-Moon Systems", + "SBN": "Small Bodies", + } + + def get_node_long_name(self, node_id): self.validate_node_id(node_id.upper()) return self.m_node_id_dict[node_id.upper()] - def validate_node_id(self,node_id): + def validate_node_id(self, node_id): if node_id.upper() not in self.m_node_id_dict: - raise UnknownNodeException(f"node_id {node_id.upper()} is not found in permissible nodes {self.m_node_id_dict.keys()}") + raise UnknownNodeException( + f"node_id {node_id.upper()} is not found in permissible nodes {self.m_node_id_dict.keys()}" + ) @classmethod def get_permissible_values(cls): return [c.lower() for c in cls.m_node_id_dict.keys()] + # @classmethod # def get_keys(cls): # return [c.lower() for c in cls.m_node_id_dict.keys()] diff --git a/src/pds_doi_service/core/input/pds4_util.py b/src/pds_doi_service/core/input/pds4_util.py index 17228fde..6467c862 100644 --- a/src/pds_doi_service/core/input/pds4_util.py +++ b/src/pds_doi_service/core/input/pds4_util.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ========== pds4_util.py @@ -12,12 +11,13 @@ Contains functions and classes for parsing PDS4 XML labels. """ - -import requests from datetime import datetime from enum import Enum -from pds_doi_service.core.entities.doi import Doi, DoiStatus, ProductType +import requests +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.input.exceptions import InputFormatException from pds_doi_service.core.util.general_util import get_logger from pds_doi_service.core.util.keyword_tokenizer import KeywordTokenizer @@ -31,39 +31,24 @@ class BestParserMethod(Enum): class DOIPDS4LabelUtil: - def __init__(self, landing_page_template): self._landing_page_template = landing_page_template self.xpath_dict = { - 'lid': - '/*/pds4:Identification_Area/pds4:logical_identifier', - 'vid': - '/*/pds4:Identification_Area/pds4:version_id', - 'title': - '/*/pds4:Identification_Area/pds4:title', - 'publication_year': - '/*/pds4:Identification_Area/pds4:Citation_Information/pds4:publication_year', - 'modification_date': - '/*/pds4:Identification_Area/pds4:Modification_History/pds4:Modification_Detail/pds4:modification_date', - 'description': - '/*/pds4:Identification_Area/pds4:Citation_Information/pds4:description', - 'product_class': - '/*/pds4:Identification_Area/pds4:product_class', - 'authors': - '/*/pds4:Identification_Area/pds4:Citation_Information/pds4:author_list', - 'editors': - '/*/pds4:Identification_Area/pds4:Citation_Information/pds4:editor_list', - 'investigation_area': - '/*/pds4:Context_Area/pds4:Investigation_Area/pds4:name', - 'observing_system_component': - '/*/pds4:Context_Area/pds4:Observing_System/pds4:Observing_System_Component/pds4:name', - 'target_identification': - '/*/pds4:Context_Area/pds4:Target_Identification/pds4:name', - 'primary_result_summary': - '/pds4:Product_Bundle/pds4:Context_Area/pds4:Primary_Result_Summary/*', - 'doi': - '/*/pds4:Identification_Area/pds4:Citation_Information/pds4:doi' + "lid": "/*/pds4:Identification_Area/pds4:logical_identifier", + "vid": "/*/pds4:Identification_Area/pds4:version_id", + "title": "/*/pds4:Identification_Area/pds4:title", + "publication_year": "/*/pds4:Identification_Area/pds4:Citation_Information/pds4:publication_year", + "modification_date": "/*/pds4:Identification_Area/pds4:Modification_History/pds4:Modification_Detail/pds4:modification_date", + "description": "/*/pds4:Identification_Area/pds4:Citation_Information/pds4:description", + "product_class": "/*/pds4:Identification_Area/pds4:product_class", + "authors": "/*/pds4:Identification_Area/pds4:Citation_Information/pds4:author_list", + "editors": "/*/pds4:Identification_Area/pds4:Citation_Information/pds4:editor_list", + "investigation_area": "/*/pds4:Context_Area/pds4:Investigation_Area/pds4:name", + "observing_system_component": "/*/pds4:Context_Area/pds4:Observing_System/pds4:Observing_System_Component/pds4:name", + "target_identification": "/*/pds4:Context_Area/pds4:Target_Identification/pds4:name", + "primary_result_summary": "/pds4:Product_Bundle/pds4:Context_Area/pds4:Primary_Result_Summary/*", + "doi": "/*/pds4:Identification_Area/pds4:Citation_Information/pds4:doi", } def is_pds4_label(self, xml_tree): @@ -102,15 +87,15 @@ def read_pds4(self, xml_tree): """ pds4_field_value_dict = {} - pds4_namespace = {'pds4': 'http://pds.nasa.gov/pds4/pds/v1'} + pds4_namespace = {"pds4": "http://pds.nasa.gov/pds4/pds/v1"} for key, xpath in self.xpath_dict.items(): elements = xml_tree.xpath(xpath, namespaces=pds4_namespace) if elements: - pds4_field_value_dict[key] = ' '.join( - [element.text.strip() - for element in elements if element.text]).strip() + pds4_field_value_dict[key] = " ".join( + [element.text.strip() for element in elements if element.text] + ).strip() return pds4_field_value_dict @@ -126,11 +111,11 @@ def _check_for_possible_full_name(self, names_list): num_person_names = 0 for one_name in names_list: - if '.' in one_name: + if "." in one_name: num_dots_found += 1 # Now that the dot is found, look to see the name contains at # least two tokens. - if len(one_name.strip().split('.')) >= 2: + if len(one_name.strip().split(".")) >= 2: # 'R. Deen' split to ['R','Deen'], "J.Maki" split to ['J','Maki'] num_person_names += 1 else: @@ -147,10 +132,11 @@ def _check_for_possible_full_name(self, names_list): if num_dots_found == len(names_list) or num_person_names == len(names_list): o_list_contains_full_name_flag = True - logger.debug(f"num_dots_found,num_person_names,len(names_list),names_list " - f"{num_dots_found,num_person_names,len(names_list),names_list}") - logger.debug(f"o_list_contains_full_name_flag " - f"{o_list_contains_full_name_flag,names_list,len(names_list)}") + logger.debug( + f"num_dots_found,num_person_names,len(names_list),names_list " + f"{num_dots_found,num_person_names,len(names_list),names_list}" + ) + logger.debug(f"o_list_contains_full_name_flag " f"{o_list_contains_full_name_flag,names_list,len(names_list)}") return o_list_contains_full_name_flag @@ -185,8 +171,8 @@ def _find_method_to_parse_authors(self, pds4_fields_authors): # The 'authors' field from data providers can be inconsistent. # Sometimes a comma ',' is used to separate the names, sometimes its # semi-colons ';' - authors_from_comma_split = pds4_fields_authors.split(',') - authors_from_semi_colon_split = pds4_fields_authors.split(';') + authors_from_comma_split = pds4_fields_authors.split(",") + authors_from_semi_colon_split = pds4_fields_authors.split(";") # Check from authors_from_comma_split to see if it possibly contains full name. # Mostly this case: "R. Deen, H. Abarca, P. Zamani, J.Maki" @@ -194,8 +180,8 @@ def _find_method_to_parse_authors(self, pds4_fields_authors): # "VanBommel, S. J., Guinness, E., Stein, T., and the MER Science Team" comma_parsed_list_contains_full_name = self._check_for_possible_full_name(authors_from_comma_split) - number_commas = pds4_fields_authors.count(',') - number_semi_colons = pds4_fields_authors.count(';') + number_commas = pds4_fields_authors.count(",") + number_semi_colons = pds4_fields_authors.count(";") if number_semi_colons == 0: if number_commas >= 1: @@ -212,83 +198,86 @@ def _find_method_to_parse_authors(self, pds4_fields_authors): # Case 3 o_best_method = BestParserMethod.BY_SEMI_COLON - logger.debug(f"o_best_method,pds4_fields_authors " - f"{o_best_method,pds4_fields_authors} " - f"number_commas,number_semi_colons " - f"{number_commas,number_semi_colons}") - logger.debug(f"len(authors_from_comma_split),len(authors_from_semi_colon_split) " - f"{len(authors_from_comma_split),len(authors_from_semi_colon_split)}") + logger.debug( + f"o_best_method,pds4_fields_authors " + f"{o_best_method,pds4_fields_authors} " + f"number_commas,number_semi_colons " + f"{number_commas,number_semi_colons}" + ) + logger.debug( + f"len(authors_from_comma_split),len(authors_from_semi_colon_split) " + f"{len(authors_from_comma_split),len(authors_from_semi_colon_split)}" + ) return o_best_method def process_pds4_fields(self, pds4_fields): try: - product_class = pds4_fields['product_class'] - product_class_suffix = product_class.split('_')[1] + product_class = pds4_fields["product_class"] + product_class_suffix = product_class.split("_")[1] - if product_class == 'Product_Document': - product_specific_type = 'technical documentation' + if product_class == "Product_Document": + product_specific_type = "technical documentation" product_type = ProductType.Text else: - product_specific_type = 'PDS4 Refereed Data ' + product_class_suffix + product_specific_type = "PDS4 Refereed Data " + product_class_suffix product_type = ProductType.Dataset site_url = self._landing_page_template.format( - product_class_suffix, requests.utils.quote(pds4_fields['lid']), - requests.utils.quote(pds4_fields['vid']) + product_class_suffix, requests.utils.quote(pds4_fields["lid"]), requests.utils.quote(pds4_fields["vid"]) ) - editors = (self.get_editor_names(pds4_fields['editors'].split(';')) - if 'editors' in pds4_fields else None) + editors = self.get_editor_names(pds4_fields["editors"].split(";")) if "editors" in pds4_fields else None # The 'authors' field is inconsistent on the use of separators. # Try to make a best guess on which method is better. - o_best_method = self._find_method_to_parse_authors(pds4_fields['authors']) + o_best_method = self._find_method_to_parse_authors(pds4_fields["authors"]) if o_best_method == BestParserMethod.BY_COMMA: - authors_list = pds4_fields['authors'].split(',') + authors_list = pds4_fields["authors"].split(",") elif o_best_method == BestParserMethod.BY_SEMI_COLON: - authors_list = pds4_fields['authors'].split(';') + authors_list = pds4_fields["authors"].split(";") else: - logger.error(f"o_best_method,pds4_fields['authors'] " - f"{o_best_method,pds4_fields['authors']}") - raise InputFormatException( - "Cannot split the authors using comma or semi-colon." - ) + logger.error(f"o_best_method,pds4_fields['authors'] " f"{o_best_method,pds4_fields['authors']}") + raise InputFormatException("Cannot split the authors using comma or semi-colon.") doi_suffix = None - if 'doi' in pds4_fields: - doi_prefix_suffix = pds4_fields['doi'].split('/') + if "doi" in pds4_fields: + doi_prefix_suffix = pds4_fields["doi"].split("/") if len(doi_prefix_suffix) == 2: doi_suffix = doi_prefix_suffix[1] timestamp = datetime.now() - identifier = pds4_fields['lid'] - - if pds4_fields['vid']: - identifier += '::' + pds4_fields['vid'] - - doi = Doi(status=DoiStatus.Unknown, - title=pds4_fields['title'], - description=pds4_fields['description'], - publication_date=self.get_publication_date(pds4_fields), - product_type=product_type, - product_type_specific=product_specific_type, - related_identifier=identifier, - site_url=site_url, - authors=self.get_author_names(authors_list), - editors=editors, - keywords=self.get_keywords(pds4_fields), - date_record_added=timestamp, - date_record_updated=timestamp, - id=doi_suffix) + identifier = pds4_fields["lid"] + + if pds4_fields["vid"]: + identifier += "::" + pds4_fields["vid"] + + doi = Doi( + status=DoiStatus.Unknown, + title=pds4_fields["title"], + description=pds4_fields["description"], + publication_date=self.get_publication_date(pds4_fields), + product_type=product_type, + product_type_specific=product_specific_type, + related_identifier=identifier, + site_url=site_url, + authors=self.get_author_names(authors_list), + editors=editors, + keywords=self.get_keywords(pds4_fields), + date_record_added=timestamp, + date_record_updated=timestamp, + id=doi_suffix, + ) except KeyError as key_err: missing_key = key_err.args[0] - msg = (f"Could not find a value for an expected PS4 label field: {key_err}.\n" - f"Please ensure there is a value present in the label for the " - f"following xpath: {self.xpath_dict[missing_key]}") + msg = ( + f"Could not find a value for an expected PS4 label field: {key_err}.\n" + f"Please ensure there is a value present in the label for the " + f"following xpath: {self.xpath_dict[missing_key]}" + ) logger.error(msg) raise InputFormatException(msg) @@ -297,7 +286,7 @@ def process_pds4_fields(self, pds4_fields): def get_publication_date(self, pds4_fields): # The field 'modification_date' is favored first. # If it is present use it, otherwise use 'publication_year' field next. - if 'modification_date' in pds4_fields: + if "modification_date" in pds4_fields: logger.debug( f"pds4_fields['modification_date'] " f"{pds4_fields['modification_date'], type(pds4_fields['modification_date'])}" @@ -305,21 +294,23 @@ def get_publication_date(self, pds4_fields): # Some PDS4 labels have more than one 'modification_date' field, # so sort in ascending and select the first date. - latest_mod_date = sorted(pds4_fields['modification_date'].split(), reverse=False)[0] - publication_date = datetime.strptime(latest_mod_date, '%Y-%m-%d') - elif 'publication_year' in pds4_fields: - publication_date = datetime.strptime(pds4_fields['publication_year'], '%Y') + latest_mod_date = sorted(pds4_fields["modification_date"].split(), reverse=False)[0] + publication_date = datetime.strptime(latest_mod_date, "%Y-%m-%d") + elif "publication_year" in pds4_fields: + publication_date = datetime.strptime(pds4_fields["publication_year"], "%Y") else: publication_date = datetime.now() return publication_date def get_keywords(self, pds4_fields): - keyword_fields = {'investigation_area', - 'observing_system_component', - 'target_identification', - 'primary_result_summary', - 'description'} + keyword_fields = { + "investigation_area", + "observing_system_component", + "target_identification", + "primary_result_summary", + "description", + } keyword_tokenizer = KeywordTokenizer() @@ -331,17 +322,17 @@ def get_keywords(self, pds4_fields): @staticmethod def _smart_first_last_name_detector(split_fullname, default_order=(0, -1)): - if len(split_fullname[0]) == 1 or split_fullname[0][-1] == '.': + if len(split_fullname[0]) == 1 or split_fullname[0][-1] == ".": return 0, -1 - elif len(split_fullname[-1]) == 1 or split_fullname[-1][-1] == '.': + elif len(split_fullname[-1]) == 1 or split_fullname[-1][-1] == ".": return -1, 0 else: return default_order @staticmethod - def _get_name_components(full_name, first_last_name_order, - first_last_name_separators, - use_smart_first_name_detector=True): + def _get_name_components( + full_name, first_last_name_order, first_last_name_separators, use_smart_first_name_detector=True + ): logger.debug(f"parse full_name {full_name}") full_name = full_name.strip() @@ -361,37 +352,33 @@ def _get_name_components(full_name, first_last_name_order, first_i, last_i = tuple(first_last_name_order) # re-add . if it has been removed as a separator - first_name_suffix = '.' if sep == '.' else '' + first_name_suffix = "." if sep == "." else "" person = { - 'first_name': split_fullname[first_i] + first_name_suffix, - 'last_name': split_fullname[last_i] + "first_name": split_fullname[first_i] + first_name_suffix, + "last_name": split_fullname[last_i], } if len(split_fullname) >= 3: - person['middle_name'] = split_fullname[1] + person["middle_name"] = split_fullname[1] break if not person: - person = {'full_name': full_name} + person = {"full_name": full_name} - logger.debug(f'parsed person {person}') + logger.debug(f"parsed person {person}") return person - def get_names(self, name_list, first_last_name_order=(0, -1), - first_last_name_separator=(',', '.')): + def get_names(self, name_list, first_last_name_order=(0, -1), first_last_name_separator=(",", ".")): logger.debug(f"name_list {name_list}") logger.debug(f"first_last_name_order {first_last_name_order}") persons = [] for full_name in name_list: - persons.append( - self._get_name_components(full_name, first_last_name_order, - first_last_name_separator) - ) + persons.append(self._get_name_components(full_name, first_last_name_order, first_last_name_separator)) return persons @@ -399,4 +386,4 @@ def get_author_names(self, name_list): return self.get_names(name_list, first_last_name_order=(-1, 0)) def get_editor_names(self, name_list): - return self.get_names(name_list, first_last_name_separator=(',',)) + return self.get_names(name_list, first_last_name_separator=(",",)) diff --git a/src/pds_doi_service/core/input/test/__init__.py b/src/pds_doi_service/core/input/test/__init__.py index 50042471..87dd0b43 100644 --- a/src/pds_doi_service/core/input/test/__init__.py +++ b/src/pds_doi_service/core/input/test/__init__.py @@ -1,12 +1,11 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for core inputs -''' - - +""" import unittest -from . import input_util_test, read_bundle + +from . import input_util_test +from . import read_bundle def suite(): diff --git a/src/pds_doi_service/core/input/test/input_util_test.py b/src/pds_doi_service/core/input/test/input_util_test.py index a7522930..4949c3a4 100644 --- a/src/pds_doi_service/core/input/test/input_util_test.py +++ b/src/pds_doi_service/core/input/test/input_util_test.py @@ -1,66 +1,65 @@ #!/usr/bin/env python - import datetime import os import unittest -from os.path import abspath, join - -from pkg_resources import resource_filename +from os.path import abspath +from os.path import join -from pds_doi_service.core.entities.doi import Doi, DoiStatus, ProductType -from pds_doi_service.core.input.input_util import DOIInputUtil +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.input.exceptions import InputFormatException -from pds_doi_service.core.outputs.service import DOIServiceFactory, SERVICE_TYPE_OSTI +from pds_doi_service.core.input.input_util import DOIInputUtil +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_OSTI +from pkg_resources import resource_filename class InputUtilTestCase(unittest.TestCase): - def setUp(self): - self.test_dir = resource_filename(__name__, '') - self.input_dir = abspath( - join(self.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + self.test_dir = resource_filename(__name__, "") + self.input_dir = abspath(join(self.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) def test_parse_dois_from_input_file(self): """Test the DOIInputUtil.parse_dois_from_input_file() method""" - doi_input_util = DOIInputUtil(valid_extensions='.xml') + doi_input_util = DOIInputUtil(valid_extensions=".xml") # Test with local file - i_filepath = join(self.input_dir, 'bundle_in_with_contributors.xml') + i_filepath = join(self.input_dir, "bundle_in_with_contributors.xml") dois = doi_input_util.parse_dois_from_input_file(i_filepath) self.assertEqual(len(dois), 1) # Test with remote file - i_filepath = 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/bundle.xml' + i_filepath = "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/bundle.xml" dois = doi_input_util.parse_dois_from_input_file(i_filepath) self.assertEqual(len(dois), 1) # Test with local directory - i_filepath = join(self.input_dir, 'draft_dir_two_files') + i_filepath = join(self.input_dir, "draft_dir_two_files") dois = doi_input_util.parse_dois_from_input_file(i_filepath) self.assertEqual(len(dois), 2) # Test with invalid local file path (does not exist) - i_filepath = '/dev/null/file/does/not/exist' + i_filepath = "/dev/null/file/does/not/exist" with self.assertRaises(InputFormatException): doi_input_util.parse_dois_from_input_file(i_filepath) # Test with invalid remote file path (does not exist) - i_filepath = 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/fake_bundle.xml' + i_filepath = "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/fake_bundle.xml" with self.assertRaises(InputFormatException): doi_input_util.parse_dois_from_input_file(i_filepath) # Test local file with invalid extension - i_filepath = join(self.input_dir, 'DOI_Reserved_GEO_200318.xlsx') + i_filepath = join(self.input_dir, "DOI_Reserved_GEO_200318.xlsx") with self.assertRaises(InputFormatException): doi_input_util.parse_dois_from_input_file(i_filepath) # Test remote file with invalid extension - doi_input_util = DOIInputUtil(valid_extensions='.csv') - i_filepath = 'https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/bundle.xml' + doi_input_util = DOIInputUtil(valid_extensions=".csv") + i_filepath = "https://pds-imaging.jpl.nasa.gov/data/nsyt/insight_cameras/bundle.xml" with self.assertRaises(InputFormatException): doi_input_util.parse_dois_from_input_file(i_filepath) @@ -69,7 +68,7 @@ def test_read_xls(self): doi_input_util = DOIInputUtil() # Test single entry spreadsheet - i_filepath = join(self.input_dir, 'DOI_Reserved_GEO_200318.xlsx') + i_filepath = join(self.input_dir, "DOI_Reserved_GEO_200318.xlsx") dois = doi_input_util.parse_xls_file(i_filepath) self.assertEqual(len(dois), 1) @@ -77,59 +76,46 @@ def test_read_xls(self): doi = dois[0] self.assertIsInstance(doi, Doi) - self.assertEqual(doi.title, 'Laboratory Shocked Feldspars Bundle') + self.assertEqual(doi.title, "Laboratory Shocked Feldspars Bundle") self.assertEqual(doi.status, DoiStatus.Reserved) - self.assertEqual(doi.related_identifier, 'urn:nasa:pds:lab_shocked_feldspars') + self.assertEqual(doi.related_identifier, "urn:nasa:pds:lab_shocked_feldspars") self.assertEqual(len(doi.authors), 1) self.assertEqual(doi.product_type, ProductType.Collection) - self.assertEqual(doi.product_type_specific, 'PDS4 Collection') + self.assertEqual(doi.product_type_specific, "PDS4 Collection") self.assertIsInstance(doi.publication_date, datetime.datetime) # Test multi entry spreadsheet - i_filepath = join( - self.input_dir, - 'DOI_Reserved_GEO_200318_with_corrected_identifier.xlsx' - ) + i_filepath = join(self.input_dir, "DOI_Reserved_GEO_200318_with_corrected_identifier.xlsx") dois = doi_input_util.parse_xls_file(i_filepath) self.assertEqual(len(dois), 3) - self.assertTrue(all([doi.title.startswith('Laboratory Shocked Feldspars') - for doi in dois])) - self.assertTrue(all([doi.status == DoiStatus.Reserved - for doi in dois])) - self.assertTrue(all([doi.related_identifier.startswith('urn:nasa:pds:lab_shocked_feldspars') - for doi in dois])) - self.assertTrue(all([len(doi.authors) == 1 - for doi in dois])) - self.assertTrue(all([doi.product_type == doi_input_util._parse_product_type(doi.product_type_specific) - for doi in dois])) - self.assertTrue(all([isinstance(doi.publication_date, datetime.datetime) - for doi in dois])) + self.assertTrue(all([doi.title.startswith("Laboratory Shocked Feldspars") for doi in dois])) + self.assertTrue(all([doi.status == DoiStatus.Reserved for doi in dois])) + self.assertTrue(all([doi.related_identifier.startswith("urn:nasa:pds:lab_shocked_feldspars") for doi in dois])) + self.assertTrue(all([len(doi.authors) == 1 for doi in dois])) + self.assertTrue( + all([doi.product_type == doi_input_util._parse_product_type(doi.product_type_specific) for doi in dois]) + ) + self.assertTrue(all([isinstance(doi.publication_date, datetime.datetime) for doi in dois])) def test_read_csv(self): """Test the DOIInputUtil.parse_csv_file() method""" doi_input_util = DOIInputUtil() - i_filepath = join(self.input_dir, 'DOI_Reserved_GEO_200318.csv') + i_filepath = join(self.input_dir, "DOI_Reserved_GEO_200318.csv") dois = doi_input_util.parse_csv_file(i_filepath) self.assertEqual(len(dois), 3) - self.assertTrue(all([doi.title.startswith('Laboratory Shocked Feldspars') - for doi in dois])) - self.assertTrue(all([doi.status == DoiStatus.Reserved - for doi in dois])) - self.assertTrue(all([doi.related_identifier.startswith('urn:nasa:pds:lab_shocked_feldspars') - for doi in dois])) - self.assertTrue(all([len(doi.authors) == 1 - for doi in dois])) - self.assertTrue(all([doi.product_type == ProductType.Collection - for doi in dois])) - self.assertTrue(all([isinstance(doi.publication_date, datetime.datetime) - for doi in dois])) + self.assertTrue(all([doi.title.startswith("Laboratory Shocked Feldspars") for doi in dois])) + self.assertTrue(all([doi.status == DoiStatus.Reserved for doi in dois])) + self.assertTrue(all([doi.related_identifier.startswith("urn:nasa:pds:lab_shocked_feldspars") for doi in dois])) + self.assertTrue(all([len(doi.authors) == 1 for doi in dois])) + self.assertTrue(all([doi.product_type == ProductType.Collection for doi in dois])) + self.assertTrue(all([isinstance(doi.publication_date, datetime.datetime) for doi in dois])) # Test on a CSV containing a PD3 style identifier - i_filepath = join(self.input_dir, 'DOI_Reserved_PDS3.csv') + i_filepath = join(self.input_dir, "DOI_Reserved_PDS3.csv") dois = doi_input_util.parse_csv_file(i_filepath) self.assertEqual(len(dois), 1) @@ -137,14 +123,14 @@ def test_read_csv(self): doi = dois[0] # Make sure the PDS3 identifier was saved off as expected - self.assertEqual(doi.related_identifier, 'LRO-L-MRFLRO-2/3/5-BISTATIC-V3.0') + self.assertEqual(doi.related_identifier, "LRO-L-MRFLRO-2/3/5-BISTATIC-V3.0") def test_read_xml(self): """Test the DOIInputUtil.parse_xml_file() method""" doi_input_util = DOIInputUtil() # Test with a PDS4 label - i_filepath = join(self.input_dir, 'bundle_in_with_contributors.xml') + i_filepath = join(self.input_dir, "bundle_in_with_contributors.xml") dois = doi_input_util.parse_xml_file(i_filepath) self.assertEqual(len(dois), 1) @@ -154,7 +140,7 @@ def test_read_xml(self): self.assertIsInstance(doi, Doi) # Test with an OSTI output label - i_filepath = join(self.input_dir, 'DOI_Release_20200727_from_reserve.xml') + i_filepath = join(self.input_dir, "DOI_Release_20200727_from_reserve.xml") dois = doi_input_util.parse_xml_file(i_filepath) self.assertEqual(len(dois), 1) @@ -164,7 +150,7 @@ def test_read_xml(self): self.assertIsInstance(doi, Doi) # Test with an OSTI label containing a PDS3 identifier - i_filepath = join(self.input_dir, 'DOI_Release_PDS3.xml') + i_filepath = join(self.input_dir, "DOI_Release_PDS3.xml") dois = doi_input_util.parse_xml_file(i_filepath) self.assertEqual(len(dois), 1) @@ -174,7 +160,7 @@ def test_read_xml(self): self.assertIsInstance(doi, Doi) # Make sure the PDS3 identifier was saved off as expected - self.assertEqual(doi.related_identifier, 'LRO-L-MRFLRO-2/3/5-BISTATIC-V3.0') + self.assertEqual(doi.related_identifier, "LRO-L-MRFLRO-2/3/5-BISTATIC-V3.0") def test_read_json(self): """Test the DOIInputUtil.parse_json_file() method""" @@ -182,9 +168,9 @@ def test_read_json(self): # Test with the appropriate JSON label for the current service if DOIServiceFactory.get_service_type() == SERVICE_TYPE_OSTI: - i_filepath = join(self.input_dir, 'DOI_Release_20210216_from_reserve.json') + i_filepath = join(self.input_dir, "DOI_Release_20210216_from_reserve.json") else: - i_filepath = join(self.input_dir, 'DOI_Release_20210615_from_reserve.json') + i_filepath = join(self.input_dir, "DOI_Release_20210615_from_reserve.json") dois = doi_input_util.parse_json_file(i_filepath) @@ -195,5 +181,5 @@ def test_read_json(self): self.assertIsInstance(doi, Doi) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/input/test/read_bundle.py b/src/pds_doi_service/core/input/test/read_bundle.py index 87429bdc..dffcfc8e 100644 --- a/src/pds_doi_service/core/input/test/read_bundle.py +++ b/src/pds_doi_service/core/input/test/read_bundle.py @@ -1,6 +1,7 @@ -import unittest import os import pathlib +import unittest + from lxml import etree @@ -8,12 +9,13 @@ class MyTestCase(unittest.TestCase): def test_read_bundle(self): test_file = os.path.join(pathlib.Path(__file__).parent.absolute(), "data", "bundle.xml") pom_doc = etree.parse(test_file) - r = pom_doc.xpath('/p:Product_Bundle/p:Identification_Area/p:Citation_Information/p:author_list', - namespaces={'p': 'http://pds.nasa.gov/pds4/pds/v1'}) + r = pom_doc.xpath( + "/p:Product_Bundle/p:Identification_Area/p:Citation_Information/p:author_list", + namespaces={"p": "http://pds.nasa.gov/pds4/pds/v1"}, + ) print(r[0].text) self.assertEqual(r[0].text, "R. Deen, H. Abarca, P. Zamani, J.Maki") - -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/input/test/read_remote_bundle.py b/src/pds_doi_service/core/input/test/read_remote_bundle.py index e662904d..d8c045c8 100644 --- a/src/pds_doi_service/core/input/test/read_remote_bundle.py +++ b/src/pds_doi_service/core/input/test/read_remote_bundle.py @@ -1,20 +1,20 @@ -import time import logging +import time logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -target_url = 'https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_JSON_1D00.JSON' +target_url = "https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_JSON_1D00.JSON" # fast method from urllib.request import urlopen timer_start = time.time() -logger.info(f"TIMER_START:urlopen {target_url}"); +logger.info(f"TIMER_START:urlopen {target_url}") response = urlopen(target_url) -logger.info("TIMER_START:reponse.read()"); -web_data = response.read().decode('utf-8'); +logger.info("TIMER_START:reponse.read()") +web_data = response.read().decode("utf-8") timer_end = time.time() timer_elapsed = timer_end - timer_start logger.info(f"TIMER_END:timer_end {timer_end}") @@ -26,15 +26,15 @@ timer_start = time.time() session = requests.session() -timer_elapsed = time.time() - timer_start; +timer_elapsed = time.time() - timer_start logger.info(f"TIMER_END:timer_end {timer_elapsed}") -response = session.get(target_url, headers={'Accept-Charset': 'utf-8'}) -timer_elapsed = time.time() - timer_start; +response = session.get(target_url, headers={"Accept-Charset": "utf-8"}) +timer_elapsed = time.time() - timer_start logger.info(f"TIMER_END:timer_end {timer_elapsed}") web_data_json = response.json() -response.encoding = 'utf-8' +response.encoding = "utf-8" web_data_str = response.text timer_end = time.time() -timer_elapsed = timer_end - timer_start; +timer_elapsed = timer_end - timer_start logger.info(f"TIMER_END:timer_end {timer_end}") logger.info(f"TIMER_ELAPSED:timer_elapsed {timer_elapsed}") diff --git a/src/pds_doi_service/core/input/test/read_xls.py b/src/pds_doi_service/core/input/test/read_xls.py index d4a45a78..2df36dfa 100644 --- a/src/pds_doi_service/core/input/test/read_xls.py +++ b/src/pds_doi_service/core/input/test/read_xls.py @@ -1,13 +1,14 @@ -import os import logging +import os + import pandas as pd logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -input_file = os.path.join(os.path.dirname(__file__), 'data', 'example-2020-04-29.xlsx') +input_file = os.path.join(os.path.dirname(__file__), "data", "example-2020-04-29.xlsx") input_df = pd.read_excel(input_file) for index, row in input_df.iterrows(): logger.info(row) - logger.info(row['status']) \ No newline at end of file + logger.info(row["status"]) diff --git a/src/pds_doi_service/core/outputs/datacite/__init__.py b/src/pds_doi_service/core/outputs/datacite/__init__.py index 978be11b..d8233805 100644 --- a/src/pds_doi_service/core/outputs/datacite/__init__.py +++ b/src/pds_doi_service/core/outputs/datacite/__init__.py @@ -6,9 +6,7 @@ This package contains the DataCite-specific implementations for the abstract classes of the outputs package. """ - from .datacite_record import DOIDataCiteRecord from .datacite_validator import DOIDataCiteValidator from .datacite_web_client import DOIDataCiteWebClient from .datacite_web_parser import DOIDataCiteWebParser - diff --git a/src/pds_doi_service/core/outputs/datacite/datacite_record.py b/src/pds_doi_service/core/outputs/datacite/datacite_record.py index 84b72cdb..5f88e5d2 100644 --- a/src/pds_doi_service/core/outputs/datacite/datacite_record.py +++ b/src/pds_doi_service/core/outputs/datacite/datacite_record.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ================== datacite_record.py @@ -13,16 +12,17 @@ Contains classes used to create DataCite-compatible labels from Doi objects in memory. """ - from os.path import exists import jinja2 -from pkg_resources import resource_filename - -from pds_doi_service.core.entities.doi import ProductType, Doi -from pds_doi_service.core.outputs.doi_record import DOIRecord, CONTENT_TYPE_JSON +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import DOIRecord from pds_doi_service.core.util.config_parser import DOIConfigUtil -from pds_doi_service.core.util.general_util import get_logger, sanitize_json_string +from pds_doi_service.core.util.general_util import get_logger +from pds_doi_service.core.util.general_util import sanitize_json_string +from pkg_resources import resource_filename logger = get_logger(__name__) @@ -34,25 +34,22 @@ class DOIDataCiteRecord(DOIRecord): This class only supports output of DOI records in JSON format. """ + def __init__(self): """Creates a new instance of DOIDataCiteRecord""" self._config = DOIConfigUtil().get_config() # Locate the jinja template - self._json_template_path = resource_filename( - __name__, 'DOI_DataCite_template_20210520-jinja2.json' - ) + self._json_template_path = resource_filename(__name__, "DOI_DataCite_template_20210520-jinja2.json") if not exists(self._json_template_path): raise RuntimeError( - 'Could not find the DOI template needed by this module\n' - f'Expected JSON template: {self._json_template_path}' + "Could not find the DOI template needed by this module\n" + f"Expected JSON template: {self._json_template_path}" ) - with open(self._json_template_path, 'r') as infile: - self._template = jinja2.Template( - infile.read(), lstrip_blocks=True, trim_blocks=True - ) + with open(self._json_template_path, "r") as infile: + self._template = jinja2.Template(infile.read(), lstrip_blocks=True, trim_blocks=True) def create_doi_record(self, dois, content_type=CONTENT_TYPE_JSON): """ @@ -73,10 +70,7 @@ def create_doi_record(self, dois, content_type=CONTENT_TYPE_JSON): """ if content_type != CONTENT_TYPE_JSON: - raise ValueError( - f'Only {CONTENT_TYPE_JSON} is supported for records created ' - f'from {__name__}' - ) + raise ValueError(f"Only {CONTENT_TYPE_JSON} is supported for records created " f"from {__name__}") # If a single DOI was provided, wrap it in a list so the iteration # below still works @@ -88,44 +82,42 @@ def create_doi_record(self, dois, content_type=CONTENT_TYPE_JSON): for doi in dois: # Filter out any keys with None as the value, so the string literal # "None" is not written out to the template - doi_fields = ( - dict(filter(lambda elem: elem[1] is not None, doi.__dict__.items())) - ) + doi_fields = dict(filter(lambda elem: elem[1] is not None, doi.__dict__.items())) # If this entry does not have a DOI assigned (i.e. reserve request), # DataCite wants to know our assigned prefix instead if not doi.doi: - doi_fields['prefix'] = self._config.get('DATACITE', 'doi_prefix') + doi_fields["prefix"] = self._config.get("DATACITE", "doi_prefix") # 'Bundle' is not supported as a product type in DataCite, so # promote to 'Collection' if doi.product_type == ProductType.Bundle: - doi_fields['product_type'] = ProductType.Collection + doi_fields["product_type"] = ProductType.Collection # Sort keywords so we can output them in the same order each time - doi_fields['keywords'] = sorted(map(sanitize_json_string, doi.keywords)) + doi_fields["keywords"] = sorted(map(sanitize_json_string, doi.keywords)) # Convert datetime objects to isoformat strings if doi.date_record_added: - doi_fields['date_record_added'] = doi.date_record_added.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + doi_fields["date_record_added"] = doi.date_record_added.strftime("%Y-%m-%dT%H:%M:%S.%fZ") if doi.date_record_updated: - doi_fields['date_record_updated'] = doi.date_record_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + doi_fields["date_record_updated"] = doi.date_record_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ") # Cleanup extra whitespace that could break JSON format from title # and description if doi.title: - doi_fields['title'] = sanitize_json_string(doi.title) + doi_fields["title"] = sanitize_json_string(doi.title) if doi.description: - doi_fields['description'] = sanitize_json_string(doi.description) + doi_fields["description"] = sanitize_json_string(doi.description) # Publication year is a must-have - doi_fields['publication_year'] = doi.publication_date.strftime('%Y') + doi_fields["publication_year"] = doi.publication_date.strftime("%Y") rendered_dois.append(doi_fields) - template_vars = {'dois': rendered_dois} + template_vars = {"dois": rendered_dois} rendered_template = self._template.render(template_vars) diff --git a/src/pds_doi_service/core/outputs/datacite/datacite_validator.py b/src/pds_doi_service/core/outputs/datacite/datacite_validator.py index 7b185670..7ba19d14 100644 --- a/src/pds_doi_service/core/outputs/datacite/datacite_validator.py +++ b/src/pds_doi_service/core/outputs/datacite/datacite_validator.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ===================== datacite_validator.py @@ -12,17 +11,15 @@ Contains functions for validating the contents of DataCite JSON labels. """ - import json +from distutils.util import strtobool from os.path import exists import jsonschema -from distutils.util import strtobool -from pkg_resources import resource_filename - from pds_doi_service.core.input.exceptions import InputFormatException from pds_doi_service.core.outputs.service_validator import DOIServiceValidator from pds_doi_service.core.util.general_util import get_logger +from pkg_resources import resource_filename logger = get_logger(__name__) @@ -36,24 +33,20 @@ class DOIDataCiteValidator(DOIServiceValidator): def __init__(self): super().__init__() - schema_file = resource_filename(__name__, 'datacite_4.3_schema.json') + schema_file = resource_filename(__name__, "datacite_4.3_schema.json") if not exists(schema_file): raise RuntimeError( - 'Could not find the schema file needed by this module.\n' - f'Expected schema file: {schema_file}' + "Could not find the schema file needed by this module.\n" f"Expected schema file: {schema_file}" ) - with open(schema_file, 'r') as infile: + with open(schema_file, "r") as infile: schema = json.load(infile) try: jsonschema.Draft7Validator.check_schema(schema) except jsonschema.exceptions.SchemaError as err: - raise RuntimeError( - f'Schema file {schema_file} is not a valid JSON schema, ' - f'reason: {err}' - ) + raise RuntimeError(f"Schema file {schema_file} is not a valid JSON schema, " f"reason: {err}") self._schema_validator = jsonschema.Draft7Validator(schema) @@ -73,17 +66,15 @@ def validate(self, label_contents): If the provided label text fails schema validation. """ - validate_against_schema = self._config.get( - 'DATACITE', 'validate_against_schema', fallback='False' - ) + validate_against_schema = self._config.get("DATACITE", "validate_against_schema", fallback="False") # Check the label contents against the DataCite JSON schema if strtobool(validate_against_schema): json_contents = json.loads(label_contents) - if 'data' in json_contents: + if "data" in json_contents: # Strip off the stuff that is not covered by the JSON schema - json_contents = json_contents['data'] + json_contents = json_contents["data"] # DataCite labels can contain a single or multiple records, # wrap single records in a list for a common interface @@ -93,19 +84,21 @@ def validate(self, label_contents): error_message = "" for index, record in enumerate(json_contents): - if 'attributes' not in record: - error_message += (f'JSON record at index {index} does not appear ' - 'to be in DataCite format.\n' - 'Please ensure the label is valid DataCite ' - 'JSON (as opposed to OSTI-format).') - elif not self._schema_validator.is_valid(record['attributes']): - error_message += (f'JSON record at index {index} does not ' - f'conform to the DataCite Schema, reason(s):\n') - - for error in self._schema_validator.iter_errors(record['attributes']): - error_message += '{path}: {message}\n'.format( - path='/'.join(map(str, error.path)), - message=error.message + if "attributes" not in record: + error_message += ( + f"JSON record at index {index} does not appear " + "to be in DataCite format.\n" + "Please ensure the label is valid DataCite " + "JSON (as opposed to OSTI-format)." + ) + elif not self._schema_validator.is_valid(record["attributes"]): + error_message += ( + f"JSON record at index {index} does not " f"conform to the DataCite Schema, reason(s):\n" + ) + + for error in self._schema_validator.iter_errors(record["attributes"]): + error_message += "{path}: {message}\n".format( + path="/".join(map(str, error.path)), message=error.message ) if error_message: diff --git a/src/pds_doi_service/core/outputs/datacite/datacite_web_client.py b/src/pds_doi_service/core/outputs/datacite/datacite_web_client.py index a1c6f474..9ad19080 100644 --- a/src/pds_doi_service/core/outputs/datacite/datacite_web_client.py +++ b/src/pds_doi_service/core/outputs/datacite/datacite_web_client.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ====================== datacite_web_client.py @@ -12,20 +11,18 @@ Contains classes used to submit labels to the DataCite DOI service endpoint. """ - import json import pprint import requests -from requests.auth import HTTPBasicAuth - from pds_doi_service.core.input.exceptions import WebRequestException -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON -from pds_doi_service.core.outputs.web_client import (DOIWebClient, - WEB_METHOD_GET, - WEB_METHOD_POST) from pds_doi_service.core.outputs.datacite.datacite_web_parser import DOIDataCiteWebParser +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.web_client import DOIWebClient +from pds_doi_service.core.outputs.web_client import WEB_METHOD_GET +from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST from pds_doi_service.core.util.general_util import get_logger +from requests.auth import HTTPBasicAuth logger = get_logger(__name__) @@ -34,14 +31,14 @@ class DOIDataCiteWebClient(DOIWebClient): """ Class used to submit HTTP requests to the DataCite DOI service. """ - _service_name = 'DataCite' + + _service_name = "DataCite" _web_parser = DOIDataCiteWebParser() - _content_type_map = { - CONTENT_TYPE_JSON: 'application/vnd.api+json' - } + _content_type_map = {CONTENT_TYPE_JSON: "application/vnd.api+json"} - def submit_content(self, payload, url=None, username=None, password=None, - method=WEB_METHOD_POST, content_type=CONTENT_TYPE_JSON): + def submit_content( + self, payload, url=None, username=None, password=None, method=WEB_METHOD_POST, content_type=CONTENT_TYPE_JSON + ): """ Submits a payload to the DataCite DOI service via the POST action. @@ -82,19 +79,18 @@ def submit_content(self, payload, url=None, username=None, password=None, response_text = super()._submit_content( payload, - url=url or config.get('DATACITE', 'url'), - username=username or config.get('DATACITE', 'user'), - password=password or config.get('DATACITE', 'password'), + url=url or config.get("DATACITE", "url"), + username=username or config.get("DATACITE", "user"), + password=password or config.get("DATACITE", "password"), method=method, - content_type=content_type + content_type=content_type, ) dois, _ = self._web_parser.parse_dois_from_label(response_text) return dois[0], response_text - def query_doi(self, query, url=None, username=None, password=None, - content_type=CONTENT_TYPE_JSON): + def query_doi(self, query, url=None, username=None, password=None, content_type=CONTENT_TYPE_JSON): """ Queries the DataCite DOI endpoint for the status of DOI submissions. @@ -133,31 +129,26 @@ def query_doi(self, query, url=None, username=None, password=None, config = self._config_util.get_config() if content_type not in self._content_type_map: - raise ValueError('Invalid content type requested, must be one of ' - f'{",".join(list(self._content_type_map.keys()))}') + raise ValueError( + "Invalid content type requested, must be one of " f'{",".join(list(self._content_type_map.keys()))}' + ) - auth = HTTPBasicAuth( - username or config.get('DATACITE', 'user'), - password or config.get('DATACITE', 'password') - ) + auth = HTTPBasicAuth(username or config.get("DATACITE", "user"), password or config.get("DATACITE", "password")) - headers = { - 'Accept': self._content_type_map[content_type] - } + headers = {"Accept": self._content_type_map[content_type]} if isinstance(query, dict): - query_string = ' '.join([f'{k}:{v}' for k, v in query.items()]) + query_string = " ".join([f"{k}:{v}" for k, v in query.items()]) else: query_string = str(query) - url = url or config.get('DATACITE', 'url') + url = url or config.get("DATACITE", "url") - logger.debug('query_string: %s', query_string) - logger.debug('url: %s', url) + logger.debug("query_string: %s", query_string) + logger.debug("url: %s", url) datacite_response = requests.request( - WEB_METHOD_GET, url=url, auth=auth, headers=headers, - params={"query": query_string} + WEB_METHOD_GET, url=url, auth=auth, headers=headers, params={"query": query_string} ) try: @@ -165,14 +156,10 @@ def query_doi(self, query, url=None, username=None, password=None, except requests.exceptions.HTTPError as http_err: # Detail text is not always present, which can cause json parsing # issues - details = ( - f'Details: {pprint.pformat(json.loads(datacite_response.text))}' - if datacite_response.text else '' - ) + details = f"Details: {pprint.pformat(json.loads(datacite_response.text))}" if datacite_response.text else "" raise WebRequestException( - 'DOI submission request to OSTI service failed, ' - f'reason: {str(http_err)}\n{details}' + "DOI submission request to OSTI service failed, " f"reason: {str(http_err)}\n{details}" ) return datacite_response.text diff --git a/src/pds_doi_service/core/outputs/datacite/datacite_web_parser.py b/src/pds_doi_service/core/outputs/datacite/datacite_web_parser.py index 89529ee5..5c949f3c 100644 --- a/src/pds_doi_service/core/outputs/datacite/datacite_web_parser.py +++ b/src/pds_doi_service/core/outputs/datacite/datacite_web_parser.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ====================== datacite_web_parser.py @@ -12,16 +11,16 @@ Contains classes used to parse response labels from DataCite DOI service requests. """ - import html import json from datetime import datetime from dateutil.parser import isoparse - -from pds_doi_service.core.entities.doi import Doi, DoiStatus, ProductType -from pds_doi_service.core.input.exceptions import (InputFormatException, - UnknownIdentifierException) +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.input.exceptions import InputFormatException +from pds_doi_service.core.input.exceptions import UnknownIdentifierException from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON from pds_doi_service.core.outputs.web_parser import DOIWebParser from pds_doi_service.core.util.general_util import get_logger @@ -36,42 +35,53 @@ class DOIDataCiteWebParser(DOIWebParser): This class only supports parsing records in JSON format. """ + _optional_fields = [ - 'id', 'doi', 'identifiers', 'description', 'keywords', 'authors', - 'site_url', 'editors', 'status', 'date_record_added', - 'date_record_updated', 'contributor' + "id", + "doi", + "identifiers", + "description", + "keywords", + "authors", + "site_url", + "editors", + "status", + "date_record_added", + "date_record_updated", + "contributor", ] _mandatory_fields = [ - 'title', 'publisher', 'publication_date', 'product_type', - 'product_type_specific', 'related_identifier' + "title", + "publisher", + "publication_date", + "product_type", + "product_type_specific", + "related_identifier", ] @staticmethod def _parse_id(record): try: - if 'suffix' in record: - return record['suffix'] + if "suffix" in record: + return record["suffix"] else: # Parse the ID from the DOI field, it it's available - return record.get('doi').split('/')[-1] + return record.get("doi").split("/")[-1] except (AttributeError, KeyError): logger.warning('Could not parse optional field "id"') @staticmethod def _parse_doi(record): try: - return record['doi'] + return record["doi"] except KeyError: logger.warning('Could not parse optional field "doi"') @staticmethod def _parse_identifiers(record): try: - identifiers = filter( - lambda identifier: identifier["identifierType"] != "DOI", - record['identifiers'] - ) + identifiers = filter(lambda identifier: identifier["identifierType"] != "DOI", record["identifiers"]) return list(identifiers) except KeyError: logger.warning('Could not parse optional field "identifiers"') @@ -79,15 +89,14 @@ def _parse_identifiers(record): @staticmethod def _parse_description(record): try: - return record['descriptions'][0]['description'] + return record["descriptions"][0]["description"] except (IndexError, KeyError): logger.warning('Could not parse optional field "description"') @staticmethod def _parse_keywords(record): try: - return set(sorted(subject['subject'] - for subject in record['subjects'])) + return set(sorted(subject["subject"] for subject in record["subjects"])) except KeyError: logger.warning('Could not parse optional field "keywords"') @@ -96,17 +105,17 @@ def _parse_authors(record): try: authors = [] - for creator in record['creators']: - if all(name_type in creator for name_type in ('givenName', 'familyName')): + for creator in record["creators"]: + if all(name_type in creator for name_type in ("givenName", "familyName")): name = f"{creator['givenName']} {creator['familyName']}" else: - name = creator['name'] + name = creator["name"] authors.append( { - 'name': name, - 'name_type': creator['nameType'], - 'name_identifiers': creator.get('nameIdentifiers', []) + "name": name, + "name_type": creator["nameType"], + "name_identifiers": creator.get("nameIdentifiers", []), } ) @@ -117,7 +126,7 @@ def _parse_authors(record): @staticmethod def _parse_site_url(record): try: - return html.unescape(record['url']) + return html.unescape(record["url"]) except (KeyError, TypeError): logger.warning('Could not parse optional field "site_url"') @@ -126,19 +135,14 @@ def _parse_editors(record): try: editors = [] - for contributor in record['contributors']: - if contributor['contributorType'] == 'Editor': - if all(name_type in contributor for name_type in ('givenName', 'familyName')): + for contributor in record["contributors"]: + if contributor["contributorType"] == "Editor": + if all(name_type in contributor for name_type in ("givenName", "familyName")): name = f"{contributor['givenName']} {contributor['familyName']}" else: - name = contributor['name'] + name = contributor["name"] - editors.append( - { - 'name': name, - 'name_identifiers': contributor.get('nameIdentifiers', []) - } - ) + editors.append({"name": name, "name_identifiers": contributor.get("nameIdentifiers", [])}) return editors except KeyError: logger.warning('Could not parse optional field "editors"') @@ -146,21 +150,21 @@ def _parse_editors(record): @staticmethod def _parse_status(record): try: - return DoiStatus(record['state']) + return DoiStatus(record["state"]) except (KeyError, ValueError): logger.warning('Could not parse optional field "status"') @staticmethod def _parse_date_record_added(record): try: - return isoparse(record['created']) + return isoparse(record["created"]) except (KeyError, ValueError): logger.warning('Could not parse optional field "date_record_added"') @staticmethod def _parse_date_record_updated(record): try: - return isoparse(record['updated']) + return isoparse(record["updated"]) except (KeyError, ValueError) as err: logger.warning('Could not parse optional field "date_record_updated"') @@ -168,15 +172,10 @@ def _parse_date_record_updated(record): def _parse_contributor(record): try: data_curator = next( - filter( - lambda contributor: contributor['contributorType'] == 'DataCurator', - record['contributors'] - ) + filter(lambda contributor: contributor["contributorType"] == "DataCurator", record["contributors"]) ) - contributor = (data_curator['name'].replace('Planetary Data System:', '') - .replace('Node', '') - .strip()) + contributor = data_curator["name"].replace("Planetary Data System:", "").replace("Node", "").strip() return contributor except (KeyError, StopIteration, ValueError): @@ -187,68 +186,56 @@ def _parse_related_identifier(record): identifier = None try: - identifier = record['relatedIdentifiers'][0]['relatedIdentifier'] + identifier = record["relatedIdentifiers"][0]["relatedIdentifier"] except (IndexError, KeyError): - if 'identifiers' in record: - for identifier_record in record['identifiers']: - if identifier_record["identifier"].startswith('urn:'): + if "identifiers" in record: + for identifier_record in record["identifiers"]: + if identifier_record["identifier"].startswith("urn:"): identifier = identifier_record["identifier"] break - elif 'url' in record: - logger.info('Parsing related identifier from URL') - identifier = DOIWebParser._get_identifier_from_site_url(record['url']) + elif "url" in record: + logger.info("Parsing related identifier from URL") + identifier = DOIWebParser._get_identifier_from_site_url(record["url"]) if identifier is None: - raise InputFormatException( - 'Failed to parse mandatory field "related_identifier"' - ) + raise InputFormatException('Failed to parse mandatory field "related_identifier"') return identifier.strip() @staticmethod def _parse_title(record): try: - return record['titles'][0]['title'] + return record["titles"][0]["title"] except (IndexError, KeyError): - raise InputFormatException( - 'Failed to parse mandatory field "title"' - ) + raise InputFormatException('Failed to parse mandatory field "title"') @staticmethod def _parse_publisher(record): try: - return record['publisher'] + return record["publisher"] except KeyError: - raise InputFormatException( - 'Failed to parse mandatory field "publisher"' - ) + raise InputFormatException('Failed to parse mandatory field "publisher"') @staticmethod def _parse_publication_date(record): try: - return datetime.strptime(str(record['publicationYear']), '%Y') + return datetime.strptime(str(record["publicationYear"]), "%Y") except (KeyError, ValueError): - raise InputFormatException( - 'Failed to parse mandatory field "publication_date"' - ) + raise InputFormatException('Failed to parse mandatory field "publication_date"') @staticmethod def _parse_product_type(record): try: - return ProductType(record['types']['resourceTypeGeneral']) + return ProductType(record["types"]["resourceTypeGeneral"]) except (KeyError, ValueError): - raise InputFormatException( - 'Failed to parse mandatory field "product_type"' - ) + raise InputFormatException('Failed to parse mandatory field "product_type"') @staticmethod def _parse_product_type_specific(record): try: - return record['types']['resourceType'] + return record["types"]["resourceType"] except KeyError: - raise InputFormatException( - 'Failed to parse mandatory field "product_type_specific"' - ) + raise InputFormatException('Failed to parse mandatory field "product_type_specific"') @staticmethod def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_JSON): @@ -272,14 +259,13 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_JSON): """ if content_type != CONTENT_TYPE_JSON: raise InputFormatException( - 'Unexpected content type provided. Value must be one of the ' - f'following: [{CONTENT_TYPE_JSON}]' + "Unexpected content type provided. Value must be one of the " f"following: [{CONTENT_TYPE_JSON}]" ) dois = [] errors = [] # DataCite does not return error information in response - datacite_records = json.loads(label_text)['data'] + datacite_records = json.loads(label_text)["data"] # DataCite can return multiple records in a list under the data key, or # a just a dictionary for a single record, make the loop work either way @@ -288,37 +274,38 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_JSON): for index, datacite_record in enumerate(datacite_records): try: - logger.info('Parsing record index %d', index) + logger.info("Parsing record index %d", index) doi_fields = {} # Everything we care about in a DataCite response is under # attributes - datacite_record = datacite_record['attributes'] + datacite_record = datacite_record["attributes"] for mandatory_field in DOIDataCiteWebParser._mandatory_fields: - doi_fields[mandatory_field] = getattr( - DOIDataCiteWebParser, f'_parse_{mandatory_field}')(datacite_record) - logger.debug('Parsed value %s for mandatory field %s', - doi_fields[mandatory_field], mandatory_field) + doi_fields[mandatory_field] = getattr(DOIDataCiteWebParser, f"_parse_{mandatory_field}")( + datacite_record + ) + logger.debug("Parsed value %s for mandatory field %s", doi_fields[mandatory_field], mandatory_field) for optional_field in DOIDataCiteWebParser._optional_fields: - parsed_value = getattr( - DOIDataCiteWebParser, f'_parse_{optional_field}')(datacite_record) + parsed_value = getattr(DOIDataCiteWebParser, f"_parse_{optional_field}")(datacite_record) if parsed_value is not None: doi_fields[optional_field] = parsed_value - logger.debug('Parsed value %s for optional field %s', - parsed_value, optional_field) + logger.debug("Parsed value %s for optional field %s", parsed_value, optional_field) doi = Doi(**doi_fields) dois.append(doi) except InputFormatException as err: - logger.warning('Failed to parse a DOI object from record index %d ' - 'of the provided label, reason: %s', index, str(err)) + logger.warning( + "Failed to parse a DOI object from record index %d " "of the provided label, reason: %s", + index, + str(err), + ) continue - logger.info('Parsed %d DOI objects from %d records', len(dois), len(datacite_records)) + logger.info("Parsed %d DOI objects from %d records", len(dois), len(datacite_records)) return dois, errors @@ -349,12 +336,12 @@ def get_record_for_identifier(label_file, identifier): If there is no record for the PDS ID in the provided label file. """ - with open(label_file, 'r') as infile: + with open(label_file, "r") as infile: records = json.load(infile) - if 'data' in records: + if "data" in records: # Strip off the stuff we don't care about - records = records['data'] + records = records["data"] # May have been handed a single record, if so wrap in a list so loop # below still works @@ -362,13 +349,12 @@ def get_record_for_identifier(label_file, identifier): records = [records] for record in records: - record_id = DOIDataCiteWebParser._parse_related_identifier(record['attributes']) + record_id = DOIDataCiteWebParser._parse_related_identifier(record["attributes"]) if record_id == identifier: # Re-add the data key we stripped off earlier - return json.dumps({'data': record}, indent=4), CONTENT_TYPE_JSON + return json.dumps({"data": record}, indent=4), CONTENT_TYPE_JSON else: raise UnknownIdentifierException( - f'Could not find entry for identifier "{identifier}" in ' - f'DataCite label file {label_file}.' + f'Could not find entry for identifier "{identifier}" in ' f"DataCite label file {label_file}." ) diff --git a/src/pds_doi_service/core/outputs/doi_record.py b/src/pds_doi_service/core/outputs/doi_record.py index fec80380..804e6625 100644 --- a/src/pds_doi_service/core/outputs/doi_record.py +++ b/src/pds_doi_service/core/outputs/doi_record.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============= doi_record.py @@ -13,8 +12,8 @@ Contains the base class for creating a record from DOI objects. """ -CONTENT_TYPE_XML = 'xml' -CONTENT_TYPE_JSON = 'json' +CONTENT_TYPE_XML = "xml" +CONTENT_TYPE_JSON = "json" """Constants for the available content types to work with""" VALID_CONTENT_TYPES = [CONTENT_TYPE_JSON, CONTENT_TYPE_XML] @@ -23,6 +22,7 @@ class DOIRecord: """Abstract base class for DOI record generating classes""" + def create_doi_record(self, dois, content_type=CONTENT_TYPE_XML): """ Creates a DOI record from the provided list of Doi objects in the @@ -42,9 +42,5 @@ def create_doi_record(self, dois, content_type=CONTENT_TYPE_XML): """ raise NotImplementedError( - f'Subclasses of {self.__class__.__name__} must provide an ' - f'implementation for create_doi_record()' + f"Subclasses of {self.__class__.__name__} must provide an " f"implementation for create_doi_record()" ) - - - diff --git a/src/pds_doi_service/core/outputs/doi_validator.py b/src/pds_doi_service/core/outputs/doi_validator.py index 83ece862..259d69e0 100644 --- a/src/pds_doi_service/core/outputs/doi_validator.py +++ b/src/pds_doi_service/core/outputs/doi_validator.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ================ doi_validator.py @@ -13,20 +12,19 @@ Contains classes and functions for validation of DOI records and the overall DOI workflow. """ - import re import requests - from pds_doi_service.core.db.doi_database import DOIDataBase -from pds_doi_service.core.entities.doi import Doi, DoiStatus -from pds_doi_service.core.input.exceptions import (DuplicatedTitleDOIException, - IllegalDOIActionException, - InvalidRecordException, - InvalidIdentifierException, - SiteURLNotExistException, - TitleDoesNotMatchProductTypeException, - UnexpectedDOIActionException) +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.input.exceptions import DuplicatedTitleDOIException +from pds_doi_service.core.input.exceptions import IllegalDOIActionException +from pds_doi_service.core.input.exceptions import InvalidIdentifierException +from pds_doi_service.core.input.exceptions import InvalidRecordException +from pds_doi_service.core.input.exceptions import SiteURLNotExistException +from pds_doi_service.core.input.exceptions import TitleDoesNotMatchProductTypeException +from pds_doi_service.core.input.exceptions import UnexpectedDOIActionException from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger @@ -52,7 +50,7 @@ class DOIValidator: DoiStatus.Pending: 4, DoiStatus.Registered: 5, DoiStatus.Findable: 5, - DoiStatus.Deactivated: 5 + DoiStatus.Deactivated: 5, } def __init__(self, db_name=None): @@ -63,7 +61,7 @@ def __init__(self, db_name=None): self.m_default_db_file = db_name else: # Default name of the database. - self.m_default_db_file = self._config.get('OTHER', 'db_file') + self.m_default_db_file = self._config.get("OTHER", "db_file") self._database_obj = DOIDataBase(self.m_default_db_file) @@ -74,12 +72,11 @@ def _check_field_site_url(self, doi: Doi): """ logger.debug("doi,site_url: %s,%s", doi, doi.site_url) - if doi.site_url and doi.site_url != 'N/A': + if doi.site_url and doi.site_url != "N/A": try: response = requests.get(doi.site_url, timeout=5) status_code = response.status_code - logger.debug("from_request status_code,site_url: %s,%s", - status_code, doi.site_url) + logger.debug("from_request status_code,site_url: %s,%s", status_code, doi.site_url) # Handle cases when a connection can be made to the server but # the status is greater than or equal to 400. @@ -96,22 +93,19 @@ def _check_field_title_duplicate(self, doi: Doi): Check if the same title exists already in local database for a different identifier. """ - query_criterias = {'title': [doi.title]} + query_criterias = {"title": [doi.title]} # Query database for rows with given title value. columns, rows = self._database_obj.select_latest_rows(query_criterias) # keep rows with same title BUT different identifier rows_with_different_identifier = [ - row for row in rows - if row[columns.index('identifier')] != doi.related_identifier + row for row in rows if row[columns.index("identifier")] != doi.related_identifier ] if rows_with_different_identifier: - identifiers = ','.join([row[columns.index('identifier')] - for row in rows_with_different_identifier]) - status = ','.join([row[columns.index('status')] - for row in rows_with_different_identifier]) + identifiers = ",".join([row[columns.index("identifier")] for row in rows_with_different_identifier]) + status = ",".join([row[columns.index("status")] for row in rows_with_different_identifier]) # Note that it is possible for rows_with_different_identifier to have # some elements while 'doi' field is None. It needs to be checked. @@ -120,14 +114,16 @@ def _check_field_title_duplicate(self, doi: Doi): # Due to the fact that 'doi' field can be None, each field must be # inspected before the join operation otherwise will cause indexing error. for row in rows_with_different_identifier: - if row[columns.index('doi')]: - dois.append(row[columns.index('doi')]) + if row[columns.index("doi")]: + dois.append(row[columns.index("doi")]) else: - dois.append('None') + dois.append("None") - msg = (f"The title '{doi.title}' has already been used for records " - f"{identifiers}, status: {status}, doi: {','.join(dois)}. " - "You must use a different title.") + msg = ( + f"The title '{doi.title}' has already been used for records " + f"{identifiers}, status: {status}, doi: {','.join(dois)}. " + "You must use a different title." + ) raise DuplicatedTitleDOIException(msg) @@ -139,22 +135,26 @@ def _check_field_title_content(self, doi: Doi): The same for: dataset, collection, document Otherwise we raise a warning. """ - product_type_specific_split = doi.product_type_specific.split(' ') + product_type_specific_split = doi.product_type_specific.split(" ") # The suffix should be the last field in the product_type_specific so # if it has many tokens, check the last one. - product_type_specific_suffix = (product_type_specific_split[-1] - if len(product_type_specific_split) > 1 - else '<<< no product specific type found >>> ') + product_type_specific_suffix = ( + product_type_specific_split[-1] + if len(product_type_specific_split) > 1 + else "<<< no product specific type found >>> " + ) logger.debug("product_type_specific_suffix: %s", product_type_specific_suffix) logger.debug("doi.title: %s", doi.title) if not product_type_specific_suffix.lower() in doi.title.lower(): - msg = (f"DOI with identifier '{doi.related_identifier}' and title " - f"'{doi.title}' does not contains the product-specific type " - f"suffix '{product_type_specific_suffix.lower()}'. " - "Product-specific type suffix should be in the title.") + msg = ( + f"DOI with identifier '{doi.related_identifier}' and title " + f"'{doi.title}' does not contains the product-specific type " + f"suffix '{product_type_specific_suffix.lower()}'. " + "Product-specific type suffix should be in the title." + ) raise TitleDoesNotMatchProductTypeException(msg) @@ -181,12 +181,12 @@ def _check_doi_for_existing_identifier(self, doi: Doi): """ # The database expects each field to be a list. - query_criterias = {'ids': [doi.related_identifier]} + query_criterias = {"ids": [doi.related_identifier]} # Query database for rows with given id value. columns, rows = self._database_obj.select_latest_rows(query_criterias) - rows_having_doi = [row for row in rows if row[columns.index('doi')]] + rows_having_doi = [row for row in rows if row[columns.index("doi")]] if rows_having_doi: pre_existing_doi = dict(zip(columns, rows_having_doi[0])) @@ -198,7 +198,7 @@ def _check_doi_for_existing_identifier(self, doi: Doi): f"(status={pre_existing_doi['status']}).\n" "You cannot update/remove a DOI for an existing record identifier." ) - elif doi.doi != pre_existing_doi['doi']: + elif doi.doi != pre_existing_doi["doi"]: raise IllegalDOIActionException( f"There is already a DOI {pre_existing_doi['doi']} submitted " f"for record identifier {doi.related_identifier} " @@ -224,13 +224,13 @@ def _check_identifier_for_existing_doi(self, doi: Doi): """ if doi.doi: # The database expects each field to be a list. - query_criterias = {'doi': [doi.doi]} + query_criterias = {"doi": [doi.doi]} # Query database for rows with given DOI value (should only ever be # at most one) columns, rows = self._database_obj.select_latest_rows(query_criterias) - if rows and doi.related_identifier != rows[0][columns.index('identifier')]: + if rows and doi.related_identifier != rows[0][columns.index("identifier")]: raise IllegalDOIActionException( f"The DOI ({doi.doi}) provided for record identifier " f"{doi.related_identifier} is already in use for record " @@ -258,20 +258,20 @@ def _check_identifier_fields(self, doi: Doi): # Make sure we have an identifier to key off of if not doi.related_identifier: raise InvalidRecordException( - 'Record provided with missing related_identifier field. ' - 'Please ensure a LIDVID or similar identifier is provided for ' - 'all DOI requests.' + "Record provided with missing related_identifier field. " + "Please ensure a LIDVID or similar identifier is provided for " + "all DOI requests." ) # Make sure the doi and id fields are consistent, if present if doi.doi and doi.id: - prefix, suffix = doi.doi.split('/') + prefix, suffix = doi.doi.split("/") if suffix != doi.id: raise InvalidRecordException( - f'Record for {doi.related_identifier} has inconsistent ' - f'DOI ({doi.doi}) and ID ({doi.id}) fields. Please reconcile ' - 'the inconsistency and resubmit the request.' + f"Record for {doi.related_identifier} has inconsistent " + f"DOI ({doi.doi}) and ID ({doi.id}) fields. Please reconcile " + "the inconsistency and resubmit the request." ) def _check_lidvid_field(self, doi: Doi): @@ -293,56 +293,56 @@ def _check_lidvid_field(self, doi: Doi): """ - if '::' in doi.related_identifier: - lid, vid = doi.related_identifier.split('::') + if "::" in doi.related_identifier: + lid, vid = doi.related_identifier.split("::") else: lid = doi.related_identifier vid = None - lid_tokens = lid.split(':') + lid_tokens = lid.split(":") try: # Make sure we got a URN - if lid_tokens[0] != 'urn': + if lid_tokens[0] != "urn": raise InvalidIdentifierException('LIDVID must start with "urn"') # Make sure we got the minimum number of fields, and that # the number of fields is consistent with the product type if not MIN_LID_FIELDS <= len(lid_tokens) <= MAX_LID_FIELDS: raise InvalidIdentifierException( - f'LIDVID must contain only between {MIN_LID_FIELDS} ' - f'and {MAX_LID_FIELDS} colon-delimited fields, ' - f'got {len(lid_tokens)} field(s)' + f"LIDVID must contain only between {MIN_LID_FIELDS} " + f"and {MAX_LID_FIELDS} colon-delimited fields, " + f"got {len(lid_tokens)} field(s)" ) # Now check each field for the expected set of characters - token_regex = re.compile(r'[a-z0-9][a-z0-9-._]{0,31}') + token_regex = re.compile(r"[a-z0-9][a-z0-9-._]{0,31}") for index, token in enumerate(lid_tokens): if not token_regex.fullmatch(token): raise InvalidIdentifierException( - f'LIDVID field {index + 1} ({token}) is invalid. ' - f'Fields must begin with a letter or digit, and only ' - f'consist of letters, digits, hyphens (-), underscores (_) ' - f'or periods (.)' + f"LIDVID field {index + 1} ({token}) is invalid. " + f"Fields must begin with a letter or digit, and only " + f"consist of letters, digits, hyphens (-), underscores (_) " + f"or periods (.)" ) # Finally, make sure the VID conforms to a version number - version_regex = re.compile(r'^\d+\.\d+$') + version_regex = re.compile(r"^\d+\.\d+$") if vid and not version_regex.fullmatch(vid): raise InvalidIdentifierException( - f'Parsed VID ({vid}) does not conform to a valid version identifier. ' - 'Version identifier must consist only of a major and minor version ' - 'joined with a period (ex: 1.0)' + f"Parsed VID ({vid}) does not conform to a valid version identifier. " + "Version identifier must consist only of a major and minor version " + "joined with a period (ex: 1.0)" ) except InvalidIdentifierException as err: raise InvalidIdentifierException( - f'The record identifier {doi.related_identifier} (DOI {doi.doi}) ' - f'does not conform to a valid LIDVID format.\n' - f'Reason: {str(err)}\n' - 'If the identifier is not intended to be a LIDVID, use the ' - 'force option to bypass the results of this check.' + f"The record identifier {doi.related_identifier} (DOI {doi.doi}) " + f"does not conform to a valid LIDVID format.\n" + f"Reason: {str(err)}\n" + "If the identifier is not intended to be a LIDVID, use the " + "force option to bypass the results of this check." ) def _check_field_workflow(self, doi: Doi): @@ -351,22 +351,24 @@ def _check_field_workflow(self, doi: Doi): identifier but a higher status than the current action (see workflow_order) """ if doi.status.lower() not in self.m_workflow_order: - msg = (f"Unexpected DOI status of '{doi.status.lower()}' from label. " - f"Valid values are " - f"{[DoiStatus(key).value for key in self.m_workflow_order.keys()]}") + msg = ( + f"Unexpected DOI status of '{doi.status.lower()}' from label. " + f"Valid values are " + f"{[DoiStatus(key).value for key in self.m_workflow_order.keys()]}" + ) logger.error(msg) raise UnexpectedDOIActionException(msg) # The database expects each field to be a list. - query_criterias = {'ids': [doi.related_identifier]} + query_criterias = {"ids": [doi.related_identifier]} # Query database for rows with given id value. columns, rows = self._database_obj.select_latest_rows(query_criterias) if rows: row = rows[0] - doi_str = row[columns.index('doi')] - prev_status = row[columns.index('status')] + doi_str = row[columns.index("doi")] + prev_status = row[columns.index("status")] # A status tuple of ('Pending',3) is higher than ('Draft',2) will # cause an error. diff --git a/src/pds_doi_service/core/outputs/osti/IAD3_schematron.sch b/src/pds_doi_service/core/outputs/osti/IAD3_schematron.sch index 00683b5c..5f9e064d 100644 --- a/src/pds_doi_service/core/outputs/osti/IAD3_schematron.sch +++ b/src/pds_doi_service/core/outputs/osti/IAD3_schematron.sch @@ -4,16 +4,16 @@ - + OSTI input document schema for 'release' action. - + - + - + if value is populated, the value in 'id' field must be exactly 7 characters long. Length of value is: @@ -24,10 +24,10 @@ If exist, publication_date field may not be empty. - + - + If exist, product_type field may not be empty. @@ -38,7 +38,7 @@ If exist, product_type field should be either 'Bundle' or 'Collection' or 'Dataset'. - + If exist, product_type_specific field may not be empty. @@ -63,4 +63,3 @@ - diff --git a/src/pds_doi_service/core/outputs/osti/__init__.py b/src/pds_doi_service/core/outputs/osti/__init__.py index f8a35c46..45ee83ba 100644 --- a/src/pds_doi_service/core/outputs/osti/__init__.py +++ b/src/pds_doi_service/core/outputs/osti/__init__.py @@ -6,10 +6,9 @@ This package contains the OSTI-specific implementations for the abstract classes of the outputs package. """ - from .osti_record import DOIOstiRecord from .osti_validator import DOIOstiValidator from .osti_web_client import DOIOstiWebClient -from .osti_web_parser import (DOIOstiWebParser, - DOIOstiJsonWebParser, - DOIOstiXmlWebParser) +from .osti_web_parser import DOIOstiJsonWebParser +from .osti_web_parser import DOIOstiWebParser +from .osti_web_parser import DOIOstiXmlWebParser diff --git a/src/pds_doi_service/core/outputs/osti/osti_record.py b/src/pds_doi_service/core/outputs/osti/osti_record.py index 813c696f..b703f9df 100644 --- a/src/pds_doi_service/core/outputs/osti/osti_record.py +++ b/src/pds_doi_service/core/outputs/osti/osti_record.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============== osti_record.py @@ -12,21 +11,20 @@ Contains classes used to create OSTI-compatible labels from Doi objects in memory. """ - import html import json from datetime import datetime from os.path import exists import pystache -from pkg_resources import resource_filename - -from pds_doi_service.core.entities.doi import Doi, ProductType -from pds_doi_service.core.outputs.doi_record import (DOIRecord, - CONTENT_TYPE_XML, - CONTENT_TYPE_JSON, - VALID_CONTENT_TYPES) +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML +from pds_doi_service.core.outputs.doi_record import DOIRecord +from pds_doi_service.core.outputs.doi_record import VALID_CONTENT_TYPES from pds_doi_service.core.util.general_util import get_logger +from pkg_resources import resource_filename logger = get_logger(__name__) @@ -38,28 +36,21 @@ class DOIOstiRecord(DOIRecord): This class supports output of DOI records in both XML and JSON format. """ + def __init__(self): """Creates a new DOIOstiRecord instance""" # Need to find the mustache DOI templates - self._xml_template_path = resource_filename( - __name__, 'DOI_IAD2_template_20200205-mustache.xml' - ) - self._json_template_path = resource_filename( - __name__, 'DOI_IAD2_template_20210216-mustache.json' - ) - - if (not exists(self._xml_template_path) - or not exists(self._json_template_path)): + self._xml_template_path = resource_filename(__name__, "DOI_IAD2_template_20200205-mustache.xml") + self._json_template_path = resource_filename(__name__, "DOI_IAD2_template_20210216-mustache.json") + + if not exists(self._xml_template_path) or not exists(self._json_template_path): raise RuntimeError( - f'Could not find one or more DOI templates needed by this module\n' - f'Expected XML template: {self._xml_template_path}\n' - f'Expected JSON template: {self._json_template_path}' + f"Could not find one or more DOI templates needed by this module\n" + f"Expected XML template: {self._xml_template_path}\n" + f"Expected JSON template: {self._json_template_path}" ) - self._template_map = { - CONTENT_TYPE_XML: self._xml_template_path, - CONTENT_TYPE_JSON: self._json_template_path - } + self._template_map = {CONTENT_TYPE_XML: self._xml_template_path, CONTENT_TYPE_JSON: self._json_template_path} def create_doi_record(self, dois, content_type=CONTENT_TYPE_XML): """ @@ -81,8 +72,7 @@ def create_doi_record(self, dois, content_type=CONTENT_TYPE_XML): """ if content_type not in VALID_CONTENT_TYPES: - raise ValueError('Invalid content type requested, must be one of ' - f'{",".join(VALID_CONTENT_TYPES)}') + raise ValueError("Invalid content type requested, must be one of " f'{",".join(VALID_CONTENT_TYPES)}') # If a single DOI was provided, wrap it in a list so the iteration # below still works @@ -94,64 +84,58 @@ def create_doi_record(self, dois, content_type=CONTENT_TYPE_XML): for index, doi in enumerate(dois): # Filter out any keys with None as the value, so the string literal # "None" is not written out as an XML tag's text body - doi_fields = ( - dict(filter(lambda elem: elem[1] is not None, doi.__dict__.items())) - ) + doi_fields = dict(filter(lambda elem: elem[1] is not None, doi.__dict__.items())) # Escape any necessary HTML characters from the site-url, # we perform this step rather than pystache to avoid # unintentional recursive escapes if doi.site_url: - doi_fields['site_url'] = html.escape(doi.site_url) + doi_fields["site_url"] = html.escape(doi.site_url) # Convert set of keywords back to a semi-colon delimited string if doi.keywords: - doi_fields['keywords'] = ";".join(sorted(doi.keywords)) + doi_fields["keywords"] = ";".join(sorted(doi.keywords)) else: - doi_fields.pop('keywords') + doi_fields.pop("keywords") # Remove any extraneous whitespace from a provided description if doi.description: - doi_fields['description'] = str.strip(doi.description) + doi_fields["description"] = str.strip(doi.description) # publication_date is assigned to a Doi object as a datetime, # need to convert to a string for the OSTI label. Note that # even if we only had the publication year from the PDS4 label, # the OSTI schema still expects YYYY-mm-dd format. if isinstance(doi.publication_date, datetime): - doi_fields['publication_date'] = doi.publication_date.strftime('%Y-%m-%d') + doi_fields["publication_date"] = doi.publication_date.strftime("%Y-%m-%d") # Same goes for date_record_added and date_record_updated - if (doi.date_record_added and - isinstance(doi.date_record_added, datetime)): - doi_fields['date_record_added'] = doi.date_record_added.strftime('%Y-%m-%d') + if doi.date_record_added and isinstance(doi.date_record_added, datetime): + doi_fields["date_record_added"] = doi.date_record_added.strftime("%Y-%m-%d") - if (doi.date_record_updated and - isinstance(doi.date_record_updated, datetime)): - doi_fields['date_record_updated'] = doi.date_record_updated.strftime('%Y-%m-%d') + if doi.date_record_updated and isinstance(doi.date_record_updated, datetime): + doi_fields["date_record_updated"] = doi.date_record_updated.strftime("%Y-%m-%d") # Pre-convert author map into a JSON string to make it play nice # with pystache rendering if doi.authors and content_type == CONTENT_TYPE_JSON: - doi_fields['authors'] = json.dumps(doi.authors) + doi_fields["authors"] = json.dumps(doi.authors) # The OSTI IAD schema does not support 'Bundle' as a product type, # so convert to collection here if doi.product_type == ProductType.Bundle: - doi_fields['product_type'] = ProductType.Collection + doi_fields["product_type"] = ProductType.Collection # Lastly, we need a kludge to inform the mustache template whether # to include a comma between consecutive entries (JSON only) if content_type == CONTENT_TYPE_JSON and index < len(dois) - 1: - doi_fields['comma'] = True + doi_fields["comma"] = True doi_fields_list.append(doi_fields) renderer = pystache.Renderer() - rendered_template = renderer.render_path( - self._template_map[content_type], {'dois': doi_fields_list} - ) + rendered_template = renderer.render_path(self._template_map[content_type], {"dois": doi_fields_list}) # Reindent the output JSON to account for the kludging of the authors field if content_type == CONTENT_TYPE_JSON: diff --git a/src/pds_doi_service/core/outputs/osti/osti_validator.py b/src/pds_doi_service/core/outputs/osti/osti_validator.py index 8761c26c..dc99d19f 100644 --- a/src/pds_doi_service/core/outputs/osti/osti_validator.py +++ b/src/pds_doi_service/core/outputs/osti/osti_validator.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ================= osti_validator.py @@ -12,21 +11,18 @@ Contains functions for validating the contents of OSTI XML labels. """ - import tempfile +from distutils.util import strtobool from os.path import exists import xmlschema -from distutils.util import strtobool -from pkg_resources import resource_filename - from lxml import etree from lxml import isoschematron - from pds_doi_service.core.entities.doi import DoiStatus from pds_doi_service.core.input.exceptions import InputFormatException from pds_doi_service.core.outputs.service_validator import DOIServiceValidator from pds_doi_service.core.util.general_util import get_logger +from pkg_resources import resource_filename logger = get_logger(__name__) @@ -40,23 +36,22 @@ class DOIOstiValidator(DOIServiceValidator): def __init__(self): super().__init__() - schematron_file = resource_filename(__name__, 'IAD3_schematron.sch') + schematron_file = resource_filename(__name__, "IAD3_schematron.sch") if not exists(schematron_file): raise RuntimeError( - 'Could not find the schematron file needed by this module.\n' - f'Expected schematron file: {schematron_file}' + "Could not find the schematron file needed by this module.\n" + f"Expected schematron file: {schematron_file}" ) sct_doc = etree.parse(schematron_file) self._schematron = isoschematron.Schematron(sct_doc, store_report=True) - xsd_filename = resource_filename(__name__, 'iad_schema.xsd') + xsd_filename = resource_filename(__name__, "iad_schema.xsd") if not exists(xsd_filename): raise RuntimeError( - 'Could not find the schema file needed by this module.\n' - f'Expected schema file: {xsd_filename}' + "Could not find the schema file needed by this module.\n" f"Expected schema file: {xsd_filename}" ) self._xsd_validator = etree.XMLSchema(file=xsd_filename) @@ -103,7 +98,7 @@ def _validate_against_xsd(self, osti_root): # error(s) occurred. if not is_valid: # Save doi_label to disk - with tempfile.NamedTemporaryFile(mode='w', suffix='temp_doi.xml') as temp_file: + with tempfile.NamedTemporaryFile(mode="w", suffix="temp_doi.xml") as temp_file: temp_file.write(etree.tostring(osti_root).decode()) temp_file.flush() @@ -145,25 +140,29 @@ def validate(self, label_contents): # Check 1. Extraneous tags in element. if len(osti_root.keys()) > 0: - msg = (f"OSTI XML cannot contain extraneous attribute(s) " - f"in main tag: {osti_root.keys()}") + msg = f"OSTI XML cannot contain extraneous attribute(s) " f"in main tag: {osti_root.keys()}" logger.error(msg) raise InputFormatException(msg) # Check 2. Bad tag(s) in element, e.g. status='Release'. # It should only be in possible_status_list variable. possible_status_list = [ - DoiStatus.Draft, DoiStatus.Reserved, DoiStatus.Reserved_not_submitted, - DoiStatus.Review, DoiStatus.Pending, DoiStatus.Registered + DoiStatus.Draft, + DoiStatus.Reserved, + DoiStatus.Reserved_not_submitted, + DoiStatus.Review, + DoiStatus.Pending, + DoiStatus.Registered, ] record_count = 1 # In the world of OSTI, record_count starts at 1. - for element in osti_root.findall('record'): - if ('status' in element.keys() - and element.attrib['status'].lower() not in possible_status_list): - msg = (f"Invalid status provided for record {record_count}. " - f"Status value must be one of {possible_status_list}. " - f"Provided {element.attrib['status'].lower()}") + for element in osti_root.findall("record"): + if "status" in element.keys() and element.attrib["status"].lower() not in possible_status_list: + msg = ( + f"Invalid status provided for record {record_count}. " + f"Status value must be one of {possible_status_list}. " + f"Provided {element.attrib['status'].lower()}" + ) logger.error(msg) raise InputFormatException(msg) @@ -171,9 +170,7 @@ def validate(self, label_contents): record_count += 1 # Determine if we need to validate against the schema as well - validate_against_schema = self._config.get( - 'OSTI', 'validate_against_schema', fallback='False' - ) + validate_against_schema = self._config.get("OSTI", "validate_against_schema", fallback="False") if strtobool(validate_against_schema): self._validate_against_xsd(osti_root) diff --git a/src/pds_doi_service/core/outputs/osti/osti_web_client.py b/src/pds_doi_service/core/outputs/osti/osti_web_client.py index f7b68f3a..ae89a779 100644 --- a/src/pds_doi_service/core/outputs/osti/osti_web_client.py +++ b/src/pds_doi_service/core/outputs/osti/osti_web_client.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ================== osti_web_client.py @@ -12,18 +11,18 @@ Contains classes used to submit labels to the OSTI DOI service endpoint. """ - import json import pprint import requests -from requests.auth import HTTPBasicAuth - from pds_doi_service.core.input.exceptions import WebRequestException -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML, CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiWebParser -from pds_doi_service.core.outputs.web_client import DOIWebClient, WEB_METHOD_POST +from pds_doi_service.core.outputs.web_client import DOIWebClient +from pds_doi_service.core.outputs.web_client import WEB_METHOD_POST from pds_doi_service.core.util.general_util import get_logger +from requests.auth import HTTPBasicAuth logger = get_logger(__name__) @@ -32,25 +31,38 @@ class DOIOstiWebClient(DOIWebClient): """ Class used to submit HTTP requests to the OSTI DOI service. """ - _service_name = 'OSTI' + + _service_name = "OSTI" _web_parser = DOIOstiWebParser() - _content_type_map = { - CONTENT_TYPE_XML: 'application/xml', - CONTENT_TYPE_JSON: 'application/json' - } + _content_type_map = {CONTENT_TYPE_XML: "application/xml", CONTENT_TYPE_JSON: "application/json"} ACCEPTABLE_FIELD_NAMES_LIST = [ - 'id', 'doi', 'accession_number', 'published_before', 'published_after', - 'added_before', 'added_after', 'updated_before', 'updated_after', - 'first_registered_before', 'first_registered_after', 'last_registered_before', - 'last_registered_after', 'status', 'start', 'rows', 'sort', 'order' + "id", + "doi", + "accession_number", + "published_before", + "published_after", + "added_before", + "added_after", + "updated_before", + "updated_after", + "first_registered_before", + "first_registered_after", + "last_registered_before", + "last_registered_after", + "status", + "start", + "rows", + "sort", + "order", ] MAX_TOTAL_ROWS_RETRIEVE = 1000000000 """Maximum numbers of rows to request from a query to OSTI""" - def submit_content(self, payload, url=None, username=None, password=None, - method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML): + def submit_content( + self, payload, url=None, username=None, password=None, method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML + ): """ Submits a payload to the OSTI DOI service via the POST action. @@ -94,11 +106,11 @@ def submit_content(self, payload, url=None, username=None, password=None, response_text = super()._submit_content( payload, - url=url or config.get('OSTI', 'url'), - username=username or config.get('OSTI', 'user'), - password=password or config.get('OSTI', 'password'), + url=url or config.get("OSTI", "url"), + username=username or config.get("OSTI", "user"), + password=password or config.get("OSTI", "password"), method=method, - content_type=content_type + content_type=content_type, ) # Re-use the parse functions from DOIOstiWebParser class to get the @@ -107,8 +119,7 @@ def submit_content(self, payload, url=None, username=None, password=None, return dois[0], response_text - def query_doi(self, query, url=None, username=None, password=None, - content_type=CONTENT_TYPE_XML): + def query_doi(self, query, url=None, username=None, password=None, content_type=CONTENT_TYPE_XML): """ Queries the status of a DOI from the OSTI server and returns the response text. @@ -141,21 +152,16 @@ def query_doi(self, query, url=None, username=None, password=None, config = self._config_util.get_config() if content_type not in self._content_type_map: - raise ValueError('Invalid content type requested, must be one of ' - f'{",".join(list(self._content_type_map.keys()))}') + raise ValueError( + "Invalid content type requested, must be one of " f'{",".join(list(self._content_type_map.keys()))}' + ) - auth = HTTPBasicAuth( - username or config.get('OSTI', 'user'), - password or config.get('OSTI', 'password') - ) + auth = HTTPBasicAuth(username or config.get("OSTI", "user"), password or config.get("OSTI", "password")) - headers = { - 'Accept': self._content_type_map[content_type], - 'Content-Type': self._content_type_map[content_type] - } + headers = {"Accept": self._content_type_map[content_type], "Content-Type": self._content_type_map[content_type]} # OSTI server requires 'rows' field to know how many max rows to fetch at once. - initial_payload = {'rows': self.MAX_TOTAL_ROWS_RETRIEVE} + initial_payload = {"rows": self.MAX_TOTAL_ROWS_RETRIEVE} # If user provided a query_dict, append to our initial payload. if query: @@ -165,29 +171,23 @@ def query_doi(self, query, url=None, username=None, password=None, else: query = initial_payload - url = url or config.get('OSTI', 'url') + url = url or config.get("OSTI", "url") logger.debug("initial_payload: %s", initial_payload) logger.debug("query_dict: %s", query) logger.debug("url: %s", url) - osti_response = requests.get( - url=url, auth=auth, params=query, headers=headers - ) + osti_response = requests.get(url=url, auth=auth, params=query, headers=headers) try: osti_response.raise_for_status() except requests.exceptions.HTTPError as http_err: # Detail text is not always present, which can cause json parsing # issues - details = ( - f'Details: {pprint.pformat(json.loads(osti_response.text))}' - if osti_response.text else '' - ) + details = f"Details: {pprint.pformat(json.loads(osti_response.text))}" if osti_response.text else "" raise WebRequestException( - 'DOI submission request to OSTI service failed, ' - f'reason: {str(http_err)}\n{details}' + "DOI submission request to OSTI service failed, " f"reason: {str(http_err)}\n{details}" ) return osti_response.text diff --git a/src/pds_doi_service/core/outputs/osti/osti_web_parser.py b/src/pds_doi_service/core/outputs/osti/osti_web_parser.py index 729ea232..bfc57b90 100644 --- a/src/pds_doi_service/core/outputs/osti/osti_web_parser.py +++ b/src/pds_doi_service/core/outputs/osti/osti_web_parser.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ================== osti_web_parser.py @@ -12,17 +11,19 @@ Contains classes used to parse response labels from OSTI DOI service requests. """ - import html import json import os from datetime import datetime from lxml import etree - -from pds_doi_service.core.entities.doi import Doi, ProductType, DoiStatus -from pds_doi_service.core.input.exceptions import InputFormatException, UnknownIdentifierException -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML, CONTENT_TYPE_JSON +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.input.exceptions import InputFormatException +from pds_doi_service.core.input.exceptions import UnknownIdentifierException +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML from pds_doi_service.core.outputs.web_parser import DOIWebParser from pds_doi_service.core.util.general_util import get_logger @@ -36,10 +37,20 @@ class DOIOstiWebParser(DOIWebParser): This class supports parsing records in both XML and JSON formats. """ + _optional_fields = [ - 'id', 'doi', 'sponsoring_organization', 'publisher', 'availability', - 'country', 'description', 'site_url', 'site_code', 'keywords', - 'authors', 'contributors' + "id", + "doi", + "sponsoring_organization", + "publisher", + "availability", + "country", + "description", + "site_url", + "site_code", + "keywords", + "authors", + "contributors", ] """The optional field names we parse from input OSTI labels.""" @@ -71,8 +82,8 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_XML): dois, errors = DOIOstiJsonWebParser.parse_dois_from_label(label_text) else: raise InputFormatException( - 'Unsupported content type provided. Value must be one of the ' - f'following: [{CONTENT_TYPE_JSON}, {CONTENT_TYPE_XML}]' + "Unsupported content type provided. Value must be one of the " + f"following: [{CONTENT_TYPE_JSON}, {CONTENT_TYPE_XML}]" ) return dois, errors @@ -107,8 +118,8 @@ def get_record_for_identifier(label_file, identifier): record = DOIOstiJsonWebParser.get_record_for_identifier(label_file, identifier) else: raise InputFormatException( - 'Unsupported file type provided. File must have one of the ' - f'following extensions: [{CONTENT_TYPE_JSON}, {CONTENT_TYPE_XML}]' + "Unsupported file type provided. File must have one of the " + f"following extensions: [{CONTENT_TYPE_JSON}, {CONTENT_TYPE_XML}]" ) return record, content_type @@ -118,6 +129,7 @@ class DOIOstiXmlWebParser(DOIOstiWebParser): """ Class used to parse OSTI-format DOI labels in XML format. """ + @staticmethod def _parse_author_names(authors_element): """ @@ -129,24 +141,21 @@ def _parse_author_names(authors_element): # If they exist, collect all the first name, middle name, last names or # full name fields into a list of dictionaries. for single_author in authors_element: - first_name = single_author.xpath('first_name') - last_name = single_author.xpath('last_name') - full_name = single_author.xpath('full_name') - middle_name = single_author.xpath('middle_name') + first_name = single_author.xpath("first_name") + last_name = single_author.xpath("last_name") + full_name = single_author.xpath("full_name") + middle_name = single_author.xpath("middle_name") author_dict = {} if full_name: - author_dict['full_name'] = full_name[0].text + author_dict["full_name"] = full_name[0].text else: if first_name and last_name: - author_dict.update( - {'first_name': first_name[0].text, - 'last_name': last_name[0].text} - ) + author_dict.update({"first_name": first_name[0].text, "last_name": last_name[0].text}) if middle_name: - author_dict.update({'middle_name': middle_name[0].text}) + author_dict.update({"middle_name": middle_name[0].text}) # It is possible that the record contains no authors. if author_dict: @@ -162,42 +171,39 @@ def _parse_contributors(contributors_element): with type "Editor". """ o_editors_list = [] - o_node_name = '' + o_node_name = "" # If they exist, collect all the editor contributor fields into a list # of dictionaries. for single_contributor in contributors_element: - first_name = single_contributor.xpath('first_name') - last_name = single_contributor.xpath('last_name') - full_name = single_contributor.xpath('full_name') - middle_name = single_contributor.xpath('middle_name') - contributor_type = single_contributor.xpath('contributor_type') + first_name = single_contributor.xpath("first_name") + last_name = single_contributor.xpath("last_name") + full_name = single_contributor.xpath("full_name") + middle_name = single_contributor.xpath("middle_name") + contributor_type = single_contributor.xpath("contributor_type") if contributor_type: - if contributor_type[0].text == 'Editor': + if contributor_type[0].text == "Editor": editor_dict = {} if full_name: - editor_dict['full_name'] = full_name[0].text + editor_dict["full_name"] = full_name[0].text else: if first_name and last_name: - editor_dict.update( - {'first_name': first_name[0].text, - 'last_name': last_name[0].text} - ) + editor_dict.update({"first_name": first_name[0].text, "last_name": last_name[0].text}) if middle_name: - editor_dict.update({'middle_name': middle_name[0].text}) + editor_dict.update({"middle_name": middle_name[0].text}) # It is possible that the record contains no contributor. if editor_dict: o_editors_list.append(editor_dict) # Parse the node ID from the name of the data curator - elif contributor_type[0].text == 'DataCurator': + elif contributor_type[0].text == "DataCurator": if full_name: o_node_name = full_name[0].text - o_node_name = o_node_name.replace('Planetary Data System:', '') - o_node_name = o_node_name.replace('Node', '') + o_node_name = o_node_name.replace("Planetary Data System:", "") + o_node_name = o_node_name.replace("Node", "") o_node_name = o_node_name.strip() else: logger.info("missing DataCurator %s", etree.tostring(single_contributor)) @@ -217,10 +223,12 @@ def _get_identifier(record): identifier = record.xpath("accession_number")[0].text elif record.xpath("related_identifiers/related_identifier[./identifier_type='URL']"): identifier = record.xpath( - "related_identifiers/related_identifier[./identifier_type='URL']/identifier_value")[0].text + "related_identifiers/related_identifier[./identifier_type='URL']/identifier_value" + )[0].text elif record.xpath("related_identifiers/related_identifier[./identifier_type='URN']"): identifier = record.xpath( - "related_identifiers/related_identifier[./identifier_type='URN']/identifier_value")[0].text + "related_identifiers/related_identifier[./identifier_type='URN']/identifier_value" + )[0].text elif record.xpath("report_numbers"): identifier = record.xpath("report_numbers")[0].text elif record.xpath("site_url"): @@ -253,44 +261,28 @@ def _parse_optional_fields(io_doi, record_element): optional_field_element = record_element.xpath(optional_field) if optional_field_element and optional_field_element[0].text is not None: - if optional_field == 'keywords': - io_doi.keywords = set(optional_field_element[0].text.split(';')) - logger.debug(f"Adding optional field 'keywords': " - f"{io_doi.keywords}") - elif optional_field == 'authors': - io_doi.authors = DOIOstiXmlWebParser._parse_author_names( - optional_field_element[0] - ) - logger.debug(f"Adding optional field 'authors': " - f"{io_doi.authors}") - elif optional_field == 'contributors': - (io_doi.editors, - io_doi.contributor) = DOIOstiXmlWebParser._parse_contributors( + if optional_field == "keywords": + io_doi.keywords = set(optional_field_element[0].text.split(";")) + logger.debug(f"Adding optional field 'keywords': " f"{io_doi.keywords}") + elif optional_field == "authors": + io_doi.authors = DOIOstiXmlWebParser._parse_author_names(optional_field_element[0]) + logger.debug(f"Adding optional field 'authors': " f"{io_doi.authors}") + elif optional_field == "contributors": + (io_doi.editors, io_doi.contributor) = DOIOstiXmlWebParser._parse_contributors( optional_field_element[0] ) - logger.debug(f"Adding optional field 'editors': " - f"{io_doi.editors}") - logger.debug(f"Adding optional field 'contributor': " - f"{io_doi.contributor}") - elif optional_field == 'date_record_added': - io_doi.date_record_added = datetime.strptime( - optional_field_element[0].text, '%Y-%m-%d' - ) - logger.debug(f"Adding optional field 'date_record_added': " - f"{io_doi.date_record_added}") - elif optional_field == 'date_record_updated': - io_doi.date_record_updated = datetime.strptime( - optional_field_element[0].text, '%Y-%m-%d' - ) - logger.debug(f"Adding optional field 'date_record_updated': " - f"{io_doi.date_record_updated}") + logger.debug(f"Adding optional field 'editors': " f"{io_doi.editors}") + logger.debug(f"Adding optional field 'contributor': " f"{io_doi.contributor}") + elif optional_field == "date_record_added": + io_doi.date_record_added = datetime.strptime(optional_field_element[0].text, "%Y-%m-%d") + logger.debug(f"Adding optional field 'date_record_added': " f"{io_doi.date_record_added}") + elif optional_field == "date_record_updated": + io_doi.date_record_updated = datetime.strptime(optional_field_element[0].text, "%Y-%m-%d") + logger.debug(f"Adding optional field 'date_record_updated': " f"{io_doi.date_record_updated}") else: setattr(io_doi, optional_field, optional_field_element[0].text) - logger.debug( - f"Adding optional field " - f"'{optional_field}': {getattr(io_doi, optional_field)}" - ) + logger.debug(f"Adding optional field " f"'{optional_field}': {getattr(io_doi, optional_field)}") return io_doi @@ -312,26 +304,23 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_XML): my_root = doc.getroottree() # Trim down input to just fields we want. - for index, record_element in enumerate(my_root.findall('record')): - status = record_element.get('status') + for index, record_element in enumerate(my_root.findall("record")): + status = record_element.get("status") if status is None: raise InputFormatException( - f'Could not parse a status for record {index + 1} from the ' - f'provided OSTI XML.' + f"Could not parse a status for record {index + 1} from the " f"provided OSTI XML." ) - if status.lower() == 'error': + if status.lower() == "error": # The 'error' record is parsed differently and does not have all # the attributes we desire. - logger.error( - f"Errors reported for record index {index + 1}" - ) + logger.error(f"Errors reported for record index {index + 1}") # Check for any errors reported back from OSTI and save # them off to be returned - errors_element = record_element.xpath('errors') - doi_message = record_element.xpath('doi_message') + errors_element = record_element.xpath("errors") + doi_message = record_element.xpath("doi_message") cur_errors = [] @@ -348,19 +337,19 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_XML): timestamp = datetime.now() - publication_date = record_element.xpath('publication_date')[0].text - product_type = record_element.xpath('product_type')[0].text - product_type_specific = record_element.xpath('product_type_specific')[0].text + publication_date = record_element.xpath("publication_date")[0].text + product_type = record_element.xpath("product_type")[0].text + product_type_specific = record_element.xpath("product_type_specific")[0].text doi = Doi( - title=record_element.xpath('title')[0].text, - publication_date=datetime.strptime(publication_date, '%Y-%m-%d'), + title=record_element.xpath("title")[0].text, + publication_date=datetime.strptime(publication_date, "%Y-%m-%d"), product_type=ProductType(product_type), product_type_specific=product_type_specific, related_identifier=identifier, status=DoiStatus(status.lower()), date_record_added=timestamp, - date_record_updated=timestamp + date_record_updated=timestamp, ) # Parse for some optional fields that may not be present in @@ -400,7 +389,7 @@ def get_record_for_identifier(label_file, identifier): """ root = etree.parse(label_file).getroot() - records = root.xpath('record') + records = root.xpath("record") for record in records: if DOIOstiXmlWebParser._get_identifier(record) == identifier: @@ -408,50 +397,42 @@ def get_record_for_identifier(label_file, identifier): break else: raise UnknownIdentifierException( - f'Could not find entry for identifier "{identifier}" in OSTI ' - f'label file {label_file}.' + f'Could not find entry for identifier "{identifier}" in OSTI ' f"label file {label_file}." ) - new_root = etree.Element('records') + new_root = etree.Element("records") new_root.append(result) - return etree.tostring( - new_root, pretty_print=True, xml_declaration=True, encoding='UTF-8' - ).decode('utf-8') + return etree.tostring(new_root, pretty_print=True, xml_declaration=True, encoding="UTF-8").decode("utf-8") class DOIOstiJsonWebParser(DOIOstiWebParser): """ Class used to parse OSTI-format DOI labels in JSON format. """ - _mandatory_fields = ['title', 'publication_date', 'product_type'] + + _mandatory_fields = ["title", "publication_date", "product_type"] @staticmethod def _parse_contributors(contributors_record): o_editors_list = list( - filter( - lambda contributor: contributor['contributor_type'] == 'Editor', - contributors_record - ) + filter(lambda contributor: contributor["contributor_type"] == "Editor", contributors_record) ) data_curator = list( - filter( - lambda contributor: contributor['contributor_type'] == 'DataCurator', - contributors_record - ) + filter(lambda contributor: contributor["contributor_type"] == "DataCurator", contributors_record) ) o_node_name = None if data_curator: - o_node_name = data_curator[0]['full_name'] - o_node_name = o_node_name.replace('Planetary Data System:', '') - o_node_name = o_node_name.replace('Node', '') + o_node_name = data_curator[0]["full_name"] + o_node_name = o_node_name.replace("Planetary Data System:", "") + o_node_name = o_node_name.replace("Node", "") o_node_name = o_node_name.strip() for editor in o_editors_list: - editor.pop('contributor_type') + editor.pop("contributor_type") return o_editors_list, o_node_name @@ -466,44 +447,30 @@ def _parse_optional_fields(io_doi, record_element): optional_field_value = record_element.get(optional_field) if optional_field_value is not None: - if optional_field == 'keywords': - io_doi.keywords = set(optional_field_value.split(';')) - logger.debug(f"Adding optional field 'keywords': " - f"{io_doi.keywords}") - elif optional_field == 'site_url': + if optional_field == "keywords": + io_doi.keywords = set(optional_field_value.split(";")) + logger.debug(f"Adding optional field 'keywords': " f"{io_doi.keywords}") + elif optional_field == "site_url": # In order to match parsing behavior of lxml, unescape # the site url io_doi.site_url = html.unescape(optional_field_value) - logger.debug(f"Adding optional field 'site_url': " - f"{io_doi.site_url}") - elif optional_field == 'contributors': - (io_doi.editors, - io_doi.contributor) = DOIOstiJsonWebParser._parse_contributors( + logger.debug(f"Adding optional field 'site_url': " f"{io_doi.site_url}") + elif optional_field == "contributors": + (io_doi.editors, io_doi.contributor) = DOIOstiJsonWebParser._parse_contributors( optional_field_value ) - logger.debug(f"Adding optional field 'editors': " - f"{io_doi.editors}") - logger.debug(f"Adding optional field 'contributor': " - f"{io_doi.contributor}") - elif optional_field == 'date_record_added': - io_doi.date_record_added = datetime.strptime( - optional_field_value, '%Y-%m-%d' - ) - logger.debug(f"Adding optional field 'date_record_added': " - f"{io_doi.date_record_added}") - elif optional_field == 'date_record_updated': - io_doi.date_record_updated = datetime.strptime( - optional_field_value, '%Y-%m-%d' - ) - logger.debug(f"Adding optional field 'date_record_updated': " - f"{io_doi.date_record_updated}") + logger.debug(f"Adding optional field 'editors': " f"{io_doi.editors}") + logger.debug(f"Adding optional field 'contributor': " f"{io_doi.contributor}") + elif optional_field == "date_record_added": + io_doi.date_record_added = datetime.strptime(optional_field_value, "%Y-%m-%d") + logger.debug(f"Adding optional field 'date_record_added': " f"{io_doi.date_record_added}") + elif optional_field == "date_record_updated": + io_doi.date_record_updated = datetime.strptime(optional_field_value, "%Y-%m-%d") + logger.debug(f"Adding optional field 'date_record_updated': " f"{io_doi.date_record_updated}") else: setattr(io_doi, optional_field, optional_field_value) - logger.debug( - f"Adding optional field " - f"'{optional_field}': {getattr(io_doi, optional_field)}" - ) + logger.debug(f"Adding optional field " f"'{optional_field}': {getattr(io_doi, optional_field)}") return io_doi @@ -554,8 +521,8 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_JSON): # Responses from OSTI come wrapped in 'records' key, strip it off # before continuing - if 'records' in osti_response: - osti_response = osti_response['records'] + if "records" in osti_response: + osti_response = osti_response["records"] # Multiple records may come in a list, or a single dict may be provided # for a single record, make the loop work either way @@ -563,27 +530,25 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_JSON): osti_response = [osti_response] for index, record in enumerate(osti_response): - if record.get('status', '').lower() == 'error': - logger.error( - f"Errors reported for record index {index + 1}" - ) + if record.get("status", "").lower() == "error": + logger.error(f"Errors reported for record index {index + 1}") # Check for any errors reported back from OSTI and save # them off to be returned cur_errors = [] - if 'errors' in record: - cur_errors.extend(record['errors']) + if "errors" in record: + cur_errors.extend(record["errors"]) - if 'doi_message' in record and len(record['doi_message']): - cur_errors.append(record['doi_message']) + if "doi_message" in record and len(record["doi_message"]): + cur_errors.append(record["doi_message"]) errors[index] = cur_errors # Make sure all the mandatory fields are present if not all([field in record for field in DOIOstiJsonWebParser._mandatory_fields]): raise InputFormatException( - 'Provided JSON is missing one or more mandatory fields: ' + "Provided JSON is missing one or more mandatory fields: " f'({", ".join(DOIOstiJsonWebParser._mandatory_fields)})' ) @@ -592,14 +557,14 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_JSON): timestamp = datetime.now() doi = Doi( - title=record['title'], - publication_date=datetime.strptime(record['publication_date'], '%Y-%m-%d'), - product_type=ProductType(record['product_type']), - product_type_specific=record.get('product_type_specific'), + title=record["title"], + publication_date=datetime.strptime(record["publication_date"], "%Y-%m-%d"), + product_type=ProductType(record["product_type"]), + product_type_specific=record.get("product_type_specific"), related_identifier=identifier, - status=DoiStatus(record.get('status', DoiStatus.Unknown).lower()), + status=DoiStatus(record.get("status", DoiStatus.Unknown).lower()), date_record_added=timestamp, - date_record_updated=timestamp + date_record_updated=timestamp, ) # Parse for some optional fields that may not be present in @@ -636,7 +601,7 @@ def get_record_for_identifier(label_file, identifier): label file. """ - with open(label_file, 'r') as infile: + with open(label_file, "r") as infile: records = json.load(infile) if not isinstance(records, list): @@ -648,8 +613,7 @@ def get_record_for_identifier(label_file, identifier): break else: raise UnknownIdentifierException( - f'Could not find entry for identifier "{identifier}" in OSTI ' - f'label file {label_file}.' + f'Could not find entry for identifier "{identifier}" in OSTI ' f"label file {label_file}." ) records = [result] diff --git a/src/pds_doi_service/core/outputs/service.py b/src/pds_doi_service/core/outputs/service.py index c7991859..2b1795e2 100644 --- a/src/pds_doi_service/core/outputs/service.py +++ b/src/pds_doi_service/core/outputs/service.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ========== service.py @@ -13,23 +12,22 @@ Contains the factory class for providing the appropriate objects based on the configured DOI service provider (OSTI, DataCite, etc...) """ - -from pds_doi_service.core.outputs.datacite import (DOIDataCiteRecord, - DOIDataCiteValidator, - DOIDataCiteWebClient, - DOIDataCiteWebParser) -from pds_doi_service.core.outputs.osti import (DOIOstiRecord, - DOIOstiValidator, - DOIOstiWebClient, - DOIOstiWebParser) +from pds_doi_service.core.outputs.datacite import DOIDataCiteRecord +from pds_doi_service.core.outputs.datacite import DOIDataCiteValidator +from pds_doi_service.core.outputs.datacite import DOIDataCiteWebClient +from pds_doi_service.core.outputs.datacite import DOIDataCiteWebParser +from pds_doi_service.core.outputs.osti import DOIOstiRecord +from pds_doi_service.core.outputs.osti import DOIOstiValidator +from pds_doi_service.core.outputs.osti import DOIOstiWebClient +from pds_doi_service.core.outputs.osti import DOIOstiWebParser from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger logger = get_logger(__name__) -SERVICE_TYPE_OSTI = 'osti' -SERVICE_TYPE_DATACITE = 'datacite' +SERVICE_TYPE_OSTI = "osti" +SERVICE_TYPE_DATACITE = "datacite" """Constants for the available service types supported by this module""" VALID_SERVICE_TYPES = [SERVICE_TYPE_OSTI, SERVICE_TYPE_DATACITE] @@ -50,28 +48,17 @@ class DOIServiceFactory: DOIServiceFactory should be necessary. """ - _DOI_RECORD_MAP = { - SERVICE_TYPE_OSTI: DOIOstiRecord, - SERVICE_TYPE_DATACITE: DOIDataCiteRecord - } + + _DOI_RECORD_MAP = {SERVICE_TYPE_OSTI: DOIOstiRecord, SERVICE_TYPE_DATACITE: DOIDataCiteRecord} """The available DOIRecord subclasses mapped to the corresponding service types""" - _SERVICE_VALIDATOR_MAP = { - SERVICE_TYPE_OSTI: DOIOstiValidator, - SERVICE_TYPE_DATACITE: DOIDataCiteValidator - } + _SERVICE_VALIDATOR_MAP = {SERVICE_TYPE_OSTI: DOIOstiValidator, SERVICE_TYPE_DATACITE: DOIDataCiteValidator} """The available DOIValidator subclasses mapped to the corresponding service types""" - _WEB_CLIENT_MAP = { - SERVICE_TYPE_OSTI: DOIOstiWebClient, - SERVICE_TYPE_DATACITE: DOIDataCiteWebClient - } + _WEB_CLIENT_MAP = {SERVICE_TYPE_OSTI: DOIOstiWebClient, SERVICE_TYPE_DATACITE: DOIDataCiteWebClient} """The available DOIWebClient subclasses mapped to the corresponding service types""" - _WEB_PARSER_MAP = { - SERVICE_TYPE_OSTI: DOIOstiWebParser, - SERVICE_TYPE_DATACITE: DOIDataCiteWebParser - } + _WEB_PARSER_MAP = {SERVICE_TYPE_OSTI: DOIOstiWebParser, SERVICE_TYPE_DATACITE: DOIDataCiteWebParser} """The available DOIWebParser subclasses mapped to the corresponding service types""" _config = DOIConfigUtil().get_config() @@ -97,8 +84,8 @@ class (or if no type is specified by the INI config at all). if service_type.lower() not in VALID_SERVICE_TYPES: raise ValueError( f'Unsupported service type "{service_type}" provided.\n' - f'Service type should be assigned to the SERVICE.provider field of ' - f'the INI config with one of the following values: {VALID_SERVICE_TYPES}' + f"Service type should be assigned to the SERVICE.provider field of " + f"the INI config with one of the following values: {VALID_SERVICE_TYPES}" ) @staticmethod @@ -113,9 +100,7 @@ def get_service_type(): by this method before it is returned. """ - service_type = DOIServiceFactory._config.get( - 'SERVICE', 'provider', fallback='unassigned' - ) + service_type = DOIServiceFactory._config.get("SERVICE", "provider", fallback="unassigned") return service_type.lower() @@ -143,8 +128,7 @@ def get_doi_record_service(service_type=None): DOIServiceFactory._check_service_type(service_type) doi_record_class = DOIServiceFactory._DOI_RECORD_MAP[service_type] - logger.debug('Returning instance of %s for service type %s', - doi_record_class.__name__, service_type) + logger.debug("Returning instance of %s for service type %s", doi_record_class.__name__, service_type) return doi_record_class() @@ -172,8 +156,7 @@ def get_validator_service(service_type=None): DOIServiceFactory._check_service_type(service_type) doi_validator_class = DOIServiceFactory._SERVICE_VALIDATOR_MAP[service_type] - logger.debug('Returning instance of %s for service type %s', - doi_validator_class.__name__, service_type) + logger.debug("Returning instance of %s for service type %s", doi_validator_class.__name__, service_type) return doi_validator_class() @@ -201,8 +184,7 @@ def get_web_client_service(service_type=None): DOIServiceFactory._check_service_type(service_type) web_client_class = DOIServiceFactory._WEB_CLIENT_MAP[service_type] - logger.debug('Returning instance of %s for service type %s', - web_client_class.__name__, service_type) + logger.debug("Returning instance of %s for service type %s", web_client_class.__name__, service_type) return web_client_class() @@ -230,7 +212,6 @@ def get_web_parser_service(service_type=None): DOIServiceFactory._check_service_type(service_type) web_parser_class = DOIServiceFactory._WEB_PARSER_MAP[service_type] - logger.debug('Returning instance of %s for service type %s', - web_parser_class.__name__, service_type) + logger.debug("Returning instance of %s for service type %s", web_parser_class.__name__, service_type) return web_parser_class() diff --git a/src/pds_doi_service/core/outputs/service_validator.py b/src/pds_doi_service/core/outputs/service_validator.py index d509acc2..ad4d599e 100644 --- a/src/pds_doi_service/core/outputs/service_validator.py +++ b/src/pds_doi_service/core/outputs/service_validator.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ==================== service_validator.py @@ -12,7 +11,6 @@ Contains the base class for creating service-specific validator objects. """ - from pds_doi_service.core.util.config_parser import DOIConfigUtil @@ -37,6 +35,5 @@ def validate(self, label_contents): """ raise NotImplementedError( - f'Subclasses of {self.__class__.__name__} must provide an ' - f'implementation for validate()' + f"Subclasses of {self.__class__.__name__} must provide an " f"implementation for validate()" ) diff --git a/src/pds_doi_service/core/outputs/test/__init__.py b/src/pds_doi_service/core/outputs/test/__init__.py index 8f50ff9f..2637e51b 100644 --- a/src/pds_doi_service/core/outputs/test/__init__.py +++ b/src/pds_doi_service/core/outputs/test/__init__.py @@ -1,11 +1,9 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for core outputs -''' - - +""" import unittest + from . import datacite_test from . import doi_validator_test from . import osti_test diff --git a/src/pds_doi_service/core/outputs/test/datacite_test.py b/src/pds_doi_service/core/outputs/test/datacite_test.py index c24d3c01..f7acb6f0 100644 --- a/src/pds_doi_service/core/outputs/test/datacite_test.py +++ b/src/pds_doi_service/core/outputs/test/datacite_test.py @@ -1,22 +1,23 @@ #!/usr/bin/env python - -import os import json -import requests +import os import unittest from datetime import datetime -from os.path import abspath, join -from requests.models import Response +from os.path import abspath +from os.path import join from unittest.mock import patch -from pkg_resources import resource_filename - -from pds_doi_service.core.entities.doi import ProductType, Doi, DoiStatus +import requests +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType from pds_doi_service.core.input.exceptions import InputFormatException -from pds_doi_service.core.outputs.datacite import (DOIDataCiteValidator, - DOIDataCiteWebParser, - DOIDataCiteWebClient, - DOIDataCiteRecord) +from pds_doi_service.core.outputs.datacite import DOIDataCiteRecord +from pds_doi_service.core.outputs.datacite import DOIDataCiteValidator +from pds_doi_service.core.outputs.datacite import DOIDataCiteWebClient +from pds_doi_service.core.outputs.datacite import DOIDataCiteWebParser +from pkg_resources import resource_filename +from requests.models import Response class DOIDataCiteRecordTestCase(unittest.TestCase): @@ -24,19 +25,15 @@ class DOIDataCiteRecordTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + cls.test_dir = resource_filename(__name__, "") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) def test_create_datacite_label_json(self): """Test creation of a DataCite JSON label from a Doi object""" # Parse sample input to obtain a Doi object - input_json_file = join( - self.input_dir, 'DOI_Release_20210615_from_reserve.json' - ) + input_json_file = join(self.input_dir, "DOI_Release_20210615_from_reserve.json") - with open(input_json_file, 'r') as infile: + with open(input_json_file, "r") as infile: input_json = infile.read() input_dois, _ = DOIDataCiteWebParser.parse_dois_from_label(input_json) @@ -55,8 +52,7 @@ def requests_valid_request_patch(method, url, **kwargs): response = Response() response.status_code = 200 - with open(join(DOIDataCiteWebClientTestCase.input_dir, - 'DOI_Release_20210615_from_release.json')) as infile: + with open(join(DOIDataCiteWebClientTestCase.input_dir, "DOI_Release_20210615_from_release.json")) as infile: response._content = infile.read().encode() return response @@ -64,28 +60,29 @@ def requests_valid_request_patch(method, url, **kwargs): class DOIDataCiteWebClientTestCase(unittest.TestCase): """Unit tests for the datacite_web_client.py module""" + input_dir = None @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + cls.test_dir = resource_filename(__name__, "") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) - @patch.object(requests, 'request', requests_valid_request_patch) + @patch.object(requests, "request", requests_valid_request_patch) def test_submit_content(self): """Test the datacite_web_client.submit_content method""" - test_doi = Doi(title='InSight Cameras Bundle', - publication_date=datetime(2019, 1, 1, 0, 0), - product_type=ProductType.Dataset, - product_type_specific='PDS4 Refereed Data Bundle', - related_identifier='urn:nasa:pds:insight_cameras::1.0', - id='yzw2-vz66', - doi='10.13143/yzw2-vz66', - publisher='NASA Planetary Data System', - contributor='Engineering', - status=DoiStatus.Reserved) + test_doi = Doi( + title="InSight Cameras Bundle", + publication_date=datetime(2019, 1, 1, 0, 0), + product_type=ProductType.Dataset, + product_type_specific="PDS4 Refereed Data Bundle", + related_identifier="urn:nasa:pds:insight_cameras::1.0", + id="yzw2-vz66", + doi="10.13143/yzw2-vz66", + publisher="NASA Planetary Data System", + contributor="Engineering", + status=DoiStatus.Reserved, + ) test_payload = DOIDataCiteRecord().create_doi_record(test_doi) @@ -103,11 +100,11 @@ def test_submit_content(self): # Check that the status has been updated by the submission request self.assertEqual(response_doi.status, DoiStatus.Findable) - @patch.object(requests, 'request', requests_valid_request_patch) + @patch.object(requests, "request", requests_valid_request_patch) def test_query_doi(self): """Test the datacite_web_client.query_doi method""" # Test with a single query term and a query dictionary - queries = ('PDS', {'id': '10.13143/yzw2-vz66'}) + queries = ("PDS", {"id": "10.13143/yzw2-vz66"}) for query in queries: response_text = DOIDataCiteWebClient().query_doi(query) @@ -116,7 +113,7 @@ def test_query_doi(self): response_doi = response_dois[0] # Should get the same record back for both queries - self.assertEqual(response_doi.doi, '10.13143/yzw2-vz66') + self.assertEqual(response_doi.doi, "10.13143/yzw2-vz66") class DOIDataCiteWebParserTestCase(unittest.TestCase): @@ -124,22 +121,40 @@ class DOIDataCiteWebParserTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) - - cls.expected_authors = [{'name': 'R. Deen', 'name_identifiers': [], 'name_type': 'Personal'}, - {'name': 'H. Abarca', 'name_identifiers': [], 'name_type': 'Personal'}, - {'name': 'P. Zamani', 'name_identifiers': [], 'name_type': 'Personal'}, - {'name': 'J. Maki', 'name_identifiers': [], 'name_type': 'Personal'}] - cls.expected_editors = [{'name': 'P. H. Smith', 'name_identifiers': []}, - {'name': 'M. Lemmon', 'name_identifiers': []}, - {'name': 'R. F. Beebe', 'name_identifiers': []}] - cls.expected_keywords = {'data', 'rdr', 'product', 'experiment', 'lander', - 'context', 'PDS', 'raw', 'mars', 'record', 'reduced', - 'science', 'edr', 'PDS4', 'camera', 'deployment', - 'insight', 'engineering'} + cls.test_dir = resource_filename(__name__, "") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) + + cls.expected_authors = [ + {"name": "R. Deen", "name_identifiers": [], "name_type": "Personal"}, + {"name": "H. Abarca", "name_identifiers": [], "name_type": "Personal"}, + {"name": "P. Zamani", "name_identifiers": [], "name_type": "Personal"}, + {"name": "J. Maki", "name_identifiers": [], "name_type": "Personal"}, + ] + cls.expected_editors = [ + {"name": "P. H. Smith", "name_identifiers": []}, + {"name": "M. Lemmon", "name_identifiers": []}, + {"name": "R. F. Beebe", "name_identifiers": []}, + ] + cls.expected_keywords = { + "data", + "rdr", + "product", + "experiment", + "lander", + "context", + "PDS", + "raw", + "mars", + "record", + "reduced", + "science", + "edr", + "PDS4", + "camera", + "deployment", + "insight", + "engineering", + } def _compare_doi_to_expected(self, doi): """ @@ -147,36 +162,34 @@ def _compare_doi_to_expected(self, doi): a parsed Doi match the expected values and/or formats. """ self.assertListEqual(doi.authors, self.expected_authors) - self.assertEqual(doi.contributor, 'Engineering') + self.assertEqual(doi.contributor, "Engineering") self.assertIsInstance(doi.date_record_added, datetime) self.assertIsInstance(doi.date_record_updated, datetime) - self.assertEqual(doi.description, - 'InSight Cameras Experiment Data Record (EDR) ' - 'and Reduced Data Record (RDR) Data Products') - self.assertEqual(doi.doi, '10.13143/yzw2-vz66') + self.assertEqual( + doi.description, + "InSight Cameras Experiment Data Record (EDR) " "and Reduced Data Record (RDR) Data Products", + ) + self.assertEqual(doi.doi, "10.13143/yzw2-vz66") self.assertListEqual(doi.editors, self.expected_editors) - self.assertEqual(doi.id, 'yzw2-vz66') + self.assertEqual(doi.id, "yzw2-vz66") self.assertSetEqual(doi.keywords, self.expected_keywords) self.assertEqual(doi.product_type, ProductType.Dataset) - self.assertEqual(doi.product_type_specific, 'PDS4 Refereed Data Bundle') + self.assertEqual(doi.product_type_specific, "PDS4 Refereed Data Bundle") self.assertIsInstance(doi.publication_date, datetime) - self.assertEqual(doi.publisher, 'NASA Planetary Data System') - self.assertEqual(doi.related_identifier, - 'urn:nasa:pds:insight_cameras::1.0') + self.assertEqual(doi.publisher, "NASA Planetary Data System") + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras::1.0") # Check that site url HTML was un-escaped as expected - self.assertIn('&', doi.site_url) - self.assertNotIn('&', doi.site_url) + self.assertIn("&", doi.site_url) + self.assertNotIn("&", doi.site_url) self.assertEqual(doi.status, DoiStatus.Draft) - self.assertEqual(doi.title, 'InSight Cameras Bundle') + self.assertEqual(doi.title, "InSight Cameras Bundle") def test_parse_osti_response_json(self): """Test parsing of an OSTI label in JSON format""" # Test with a nominal file containing most of the optional fields - input_json_file = join( - self.input_dir, 'DOI_Release_20210615_from_reserve.json' - ) + input_json_file = join(self.input_dir, "DOI_Release_20210615_from_reserve.json") - with open(input_json_file, 'r') as infile: + with open(input_json_file, "r") as infile: input_json = infile.read() dois, errors = DOIDataCiteWebParser.parse_dois_from_label(input_json) @@ -193,22 +206,18 @@ class DOIDataCiteValidatorTestCast(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + cls.test_dir = resource_filename(__name__, "") + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) def test_json_label_validation(self): """Test validation against a DataCite label created from a valid Doi object""" validator = DOIDataCiteValidator() # Parse sample input to obtain a Doi object - input_json_file = join( - self.input_dir, 'DOI_Release_20210615_from_reserve.json' - ) + input_json_file = join(self.input_dir, "DOI_Release_20210615_from_reserve.json") # Next, create a valid output DataCite label from the parsed Doi - with open(input_json_file, 'r') as infile: + with open(input_json_file, "r") as infile: input_json = infile.read() input_dois, _ = DOIDataCiteWebParser.parse_dois_from_label(input_json) @@ -219,20 +228,20 @@ def test_json_label_validation(self): # Now remove some required fields to ensure its caught by validation output_json = json.loads(output_json) - output_json['data']['attributes'].pop('publicationYear') - output_json['data']['attributes'].pop('schemaVersion') + output_json["data"]["attributes"].pop("publicationYear") + output_json["data"]["attributes"].pop("schemaVersion") output_json = json.dumps(output_json) try: validator.validate(output_json) # Should never make it here - self.fail('Invalid JSON was accepted by DOIDataCiteValidator') + self.fail("Invalid JSON was accepted by DOIDataCiteValidator") except InputFormatException as err: # Make sure the error details the reasons we expect self.assertIn("'publicationYear' is a required property", str(err)) self.assertIn("'schemaVersion' is a required property", str(err)) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/outputs/test/doi_validator_test.py b/src/pds_doi_service/core/outputs/test/doi_validator_test.py index 8635b880..e95e2921 100644 --- a/src/pds_doi_service/core/outputs/test/doi_validator_test.py +++ b/src/pds_doi_service/core/outputs/test/doi_validator_test.py @@ -5,54 +5,61 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - import datetime -import unittest import os +import unittest from pds_doi_service.core.db.doi_database import DOIDataBase -from pds_doi_service.core.entities.doi import Doi, DoiStatus, ProductType -from pds_doi_service.core.input.exceptions import (DuplicatedTitleDOIException, - InvalidRecordException, - InvalidIdentifierException, - TitleDoesNotMatchProductTypeException, - IllegalDOIActionException, - UnexpectedDOIActionException) - +from pds_doi_service.core.entities.doi import Doi +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.input.exceptions import DuplicatedTitleDOIException +from pds_doi_service.core.input.exceptions import IllegalDOIActionException +from pds_doi_service.core.input.exceptions import InvalidIdentifierException +from pds_doi_service.core.input.exceptions import InvalidRecordException +from pds_doi_service.core.input.exceptions import TitleDoesNotMatchProductTypeException +from pds_doi_service.core.input.exceptions import UnexpectedDOIActionException from pds_doi_service.core.outputs.doi_validator import DOIValidator class DoiValidatorTest(unittest.TestCase): # This file is removed in the beginning and at the end of these tests. - db_name = 'doi_temp_for_doi_validator_unit_test.db' + db_name = "doi_temp_for_doi_validator_unit_test.db" def setUp(self): self._database_obj = DOIDataBase(self.db_name) self._doi_validator = DOIValidator(db_name=self.db_name) - self.lid = 'urn:nasa:pds:lab_shocked_feldspars' - self.vid = '1.0' - self.identifier = self.lid + '::' + self.vid - self.transaction_key = './transaction_history/img/2020-06-15T18:42:45.653317' + self.lid = "urn:nasa:pds:lab_shocked_feldspars" + self.vid = "1.0" + self.identifier = self.lid + "::" + self.vid + self.transaction_key = "./transaction_history/img/2020-06-15T18:42:45.653317" self.release_date = datetime.datetime.now() self.transaction_date = datetime.datetime.now() self.status = DoiStatus.Draft - self.title = 'Laboratory Shocked Feldspars Collection' + self.title = "Laboratory Shocked Feldspars Collection" self.product_type = ProductType.Collection - self.product_type_specific = 'PDS4 Collection' - self.submitter = 'test-submitter@jpl.nasa.gov' - self.discipline_node = 'img' + self.product_type_specific = "PDS4 Collection" + self.submitter = "test-submitter@jpl.nasa.gov" + self.discipline_node = "img" - self.id = '21940' - self.doi = '10.17189/' + self.id + self.id = "21940" + self.doi = "10.17189/" + self.id # Write a record into database # All fields are valid. self._database_obj.write_doi_info_to_database( - self.identifier, self.transaction_key, self.doi, - self.release_date, self.transaction_date, - self.status, self.title, self.product_type, - self.product_type_specific, self.submitter, self.discipline_node + self.identifier, + self.transaction_key, + self.doi, + self.release_date, + self.transaction_date, + self.status, + self.title, + self.product_type, + self.product_type_specific, + self.submitter, + self.discipline_node, ) def tearDown(self): @@ -66,17 +73,17 @@ def test_existing_title_new_lidvid_exception(self): Expecting a DuplicatedTitleDOIException: titles should not be reused between different LIDVIDs. """ - doi_obj = Doi(title=self.title, - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + '1.1', - status=self.status) - - self.assertRaises( - DuplicatedTitleDOIException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title, + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + "1.1", + status=self.status, ) + self.assertRaises(DuplicatedTitleDOIException, self._doi_validator.validate, doi_obj) + def test_new_title_existing_doi_and_lidvid_nominal(self): """ Test validation of a Doi object with a new title but an existing DOI @@ -84,14 +91,16 @@ def test_new_title_existing_doi_and_lidvid_nominal(self): Expecting no error: Records may update their title as long as the DOI/LIDVID have not changed. """ - doi_obj = Doi(title=self.title + ' (NEW)', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id=self.id, - doi=self.doi, - status=self.status) + doi_obj = Doi( + title=self.title + " (NEW)", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id=self.id, + doi=self.doi, + status=self.status, + ) self._doi_validator.validate(doi_obj) @@ -102,17 +111,17 @@ def test_new_title_existing_lidvid_exception(self): Expecting IllegalDOIActionException: cannot remove a DOI associated to an existing LIDVID. """ - doi_obj = Doi(title=self.title + ' (NEW)', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - status=self.status) - - self.assertRaises( - IllegalDOIActionException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title + " (NEW)", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + status=self.status, ) + self.assertRaises(IllegalDOIActionException, self._doi_validator.validate, doi_obj) + def test_title_does_not_match_product_type_exception(self): """ Test validation of DOI with a non-matching title, existing DOI and @@ -120,19 +129,19 @@ def test_title_does_not_match_product_type_exception(self): Expecting TitleDoesNotMatchProductTypeException: product type is expected to be included with a title. """ - doi_obj = Doi(title='test title', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id=self.id, - doi=self.doi, - status=self.status) - - self.assertRaises( - TitleDoesNotMatchProductTypeException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title="test title", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id=self.id, + doi=self.doi, + status=self.status, ) + self.assertRaises(TitleDoesNotMatchProductTypeException, self._doi_validator.validate, doi_obj) + def test_title_matches_product_type_nominal(self): """ Test validation of DOI with existing DOI, and existing @@ -140,14 +149,16 @@ def test_title_matches_product_type_nominal(self): Expecting no error: title updates are allowed for existing DOI/LIDVID pairs, and title aligns with assigned product type. """ - doi_obj = Doi(title='test title ' + self.product_type_specific, - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id=self.id, - doi=self.doi, - status=self.status.lower()) + doi_obj = Doi( + title="test title " + self.product_type_specific, + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id=self.id, + doi=self.doi, + status=self.status.lower(), + ) self._doi_validator.validate(doi_obj) @@ -158,19 +169,19 @@ def test_existing_lidvid_new_doi_exception(self): Expecting IllegalDOIActionException: Each LIDVID may only be associated to a single DOI value. """ - doi_obj = Doi(title=self.title + ' different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id=self.id, - doi=self.doi + '_new_doi', - status=self.status) - - self.assertRaises( - IllegalDOIActionException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title + " different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id=self.id, + doi=self.doi + "_new_doi", + status=self.status, ) + self.assertRaises(IllegalDOIActionException, self._doi_validator.validate, doi_obj) + def test_existing_doi_new_lidvid_exception(self): """ Test validation of a Doi object with an existing title, existing DOI, and @@ -178,19 +189,19 @@ def test_existing_doi_new_lidvid_exception(self): Expecting IllegalDOIActionException: DOI may only be associated to a single LIDVID. """ - doi_obj = Doi(title=self.title + ' different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + '2.0', - id=self.id, - doi=self.doi, - status=self.status) - - self.assertRaises( - IllegalDOIActionException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title + " different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + "2.0", + id=self.id, + doi=self.doi, + status=self.status, ) + self.assertRaises(IllegalDOIActionException, self._doi_validator.validate, doi_obj) + def test_workflow_sequence_exception(self): """ Test validation of Doi object with new title, existing DOI and @@ -199,19 +210,19 @@ def test_workflow_sequence_exception(self): Expecting UnexpectedDOIActionException: Reserve step is upstream of Draft step. """ - doi_obj = Doi(title=self.title + 'different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id=self.id, - doi=self.doi, - status=DoiStatus.Reserved) - - self.assertRaises( - UnexpectedDOIActionException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title + "different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id=self.id, + doi=self.doi, + status=DoiStatus.Reserved, ) + self.assertRaises(UnexpectedDOIActionException, self._doi_validator.validate, doi_obj) + def test_workflow_sequence_nominal(self): """ Test validation of Doi object with new title, existing DOI and @@ -219,14 +230,16 @@ def test_workflow_sequence_nominal(self): entry is in 'draft'. Expecting no error: Registered step is downstream from Draft. """ - doi_obj = Doi(title=self.title + 'different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id=self.id, - doi=self.doi, - status=DoiStatus.Registered) + doi_obj = Doi( + title=self.title + "different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id=self.id, + doi=self.doi, + status=DoiStatus.Registered, + ) self._doi_validator.validate(doi_obj) @@ -236,71 +249,61 @@ def test_identifier_validation_missing_related_identifier(self): Expecting InvalidRecordException: Doi objects must always specify a related identifier to be valid. """ - doi_obj = Doi(title=self.title + ' different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier='', - id=self.id + '123', - doi=self.doi + '123', - status=DoiStatus.Reserved_not_submitted) - - self.assertRaises( - InvalidRecordException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title + " different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier="", + id=self.id + "123", + doi=self.doi + "123", + status=DoiStatus.Reserved_not_submitted, ) + self.assertRaises(InvalidRecordException, self._doi_validator.validate, doi_obj) + def test_identifier_validation_invalid_lidvid(self): """ Test validation of Doi object with various invalid LIDVIDs. Expecting InvalidLIDVIDException for each test. """ - doi_obj = Doi(title=self.title + ' different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier='', - status=DoiStatus.Reserved_not_submitted) + doi_obj = Doi( + title=self.title + " different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier="", + status=DoiStatus.Reserved_not_submitted, + ) # Test invalid starting token (must be urn) - doi_obj.related_identifier = 'url:nasa:pds:lab_shocked_feldspars::1.0' + doi_obj.related_identifier = "url:nasa:pds:lab_shocked_feldspars::1.0" - self.assertRaises( - InvalidIdentifierException, self._doi_validator.validate, doi_obj - ) + self.assertRaises(InvalidIdentifierException, self._doi_validator.validate, doi_obj) # Test invalid number of tokens (too few) - doi_obj.related_identifier = 'url:nasa:pds::1.0' + doi_obj.related_identifier = "url:nasa:pds::1.0" - self.assertRaises( - InvalidIdentifierException, self._doi_validator.validate, doi_obj - ) + self.assertRaises(InvalidIdentifierException, self._doi_validator.validate, doi_obj) # Test invalid number of tokens (too many) - doi_obj.related_identifier = 'url:nasa:pds:lab_shocked_feldspars:collection_1:product_1:dataset_1::1.0' + doi_obj.related_identifier = "url:nasa:pds:lab_shocked_feldspars:collection_1:product_1:dataset_1::1.0" - self.assertRaises( - InvalidIdentifierException, self._doi_validator.validate, doi_obj - ) + self.assertRaises(InvalidIdentifierException, self._doi_validator.validate, doi_obj) # Test invalid field tokens (invalid characters) - doi_obj.related_identifier = 'urn:nasa:_pds:lab_shocked_feldspars' + doi_obj.related_identifier = "urn:nasa:_pds:lab_shocked_feldspars" - self.assertRaises( - InvalidIdentifierException, self._doi_validator.validate, doi_obj - ) + self.assertRaises(InvalidIdentifierException, self._doi_validator.validate, doi_obj) - doi_obj.related_identifier = 'urn:nasa:pds:lab_$hocked_feldspars' + doi_obj.related_identifier = "urn:nasa:pds:lab_$hocked_feldspars" - self.assertRaises( - InvalidIdentifierException, self._doi_validator.validate, doi_obj - ) + self.assertRaises(InvalidIdentifierException, self._doi_validator.validate, doi_obj) # Test invalid VID - doi_obj.related_identifier = 'urn:nasa:pds:lab_shocked_feldspars::v1.0' + doi_obj.related_identifier = "urn:nasa:pds:lab_shocked_feldspars::v1.0" - self.assertRaises( - InvalidIdentifierException, self._doi_validator.validate, doi_obj - ) + self.assertRaises(InvalidIdentifierException, self._doi_validator.validate, doi_obj) def test_identifier_validation_doi_id_mismatch(self): """ @@ -308,19 +311,19 @@ def test_identifier_validation_doi_id_mismatch(self): Expecting InvalidRecordException: doi and id fields should always be consistent. """ - doi_obj = Doi(title=self.title + ' different', - publication_date=self.transaction_date, - product_type=self.product_type, - product_type_specific=self.product_type_specific, - related_identifier=self.lid + '::' + self.vid, - id='1234', - doi=self.doi, - status=DoiStatus.Reserved_not_submitted) - - self.assertRaises( - InvalidRecordException, self._doi_validator.validate, doi_obj + doi_obj = Doi( + title=self.title + " different", + publication_date=self.transaction_date, + product_type=self.product_type, + product_type_specific=self.product_type_specific, + related_identifier=self.lid + "::" + self.vid, + id="1234", + doi=self.doi, + status=DoiStatus.Reserved_not_submitted, ) + self.assertRaises(InvalidRecordException, self._doi_validator.validate, doi_obj) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/outputs/test/osti_test.py b/src/pds_doi_service/core/outputs/test/osti_test.py index fa52d6be..29dfd72e 100644 --- a/src/pds_doi_service/core/outputs/test/osti_test.py +++ b/src/pds_doi_service/core/outputs/test/osti_test.py @@ -1,17 +1,19 @@ #!/usr/bin/env python - import json import os -from datetime import datetime -from os.path import abspath, join import unittest +from datetime import datetime +from os.path import abspath +from os.path import join -from pkg_resources import resource_filename - -from pds_doi_service.core.entities.doi import ProductType, DoiStatus +from pds_doi_service.core.entities.doi import DoiStatus +from pds_doi_service.core.entities.doi import ProductType +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML from pds_doi_service.core.outputs.osti.osti_record import DOIOstiRecord -from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiXmlWebParser, DOIOstiJsonWebParser -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML, CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiJsonWebParser +from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiXmlWebParser +from pkg_resources import resource_filename class DOIOstiRecordTestCase(unittest.TestCase): @@ -19,29 +21,23 @@ class DOIOstiRecordTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') + cls.test_dir = resource_filename(__name__, "") # FIXME: moving this code from PACKAGE-DIR to PACKAGE-DIR/src (and, when we add # namepsace packages, to PACKAGE-DIR/src/pds) shouldn't necessitate re-jiggering # all the parent directories: - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) def test_create_osti_label_xml(self): """Test creation of an OSTI XML label from a Doi object""" # Parse sample input to obtain a Doi object - input_xml_file = join( - self.input_dir, 'DOI_Release_20200727_from_release.xml' - ) + input_xml_file = join(self.input_dir, "DOI_Release_20200727_from_release.xml") - with open(input_xml_file, 'r') as infile: + with open(input_xml_file, "r") as infile: input_xml = infile.read() input_dois, _ = DOIOstiXmlWebParser.parse_dois_from_label(input_xml) # Now create an output label from the parsed Doi - output_xml = DOIOstiRecord().create_doi_record( - input_dois[0], content_type=CONTENT_TYPE_XML - ) + output_xml = DOIOstiRecord().create_doi_record(input_dois[0], content_type=CONTENT_TYPE_XML) output_dois, _ = DOIOstiXmlWebParser.parse_dois_from_label(output_xml) # Massage the output a bit so we can do a straight dict comparison @@ -50,7 +46,7 @@ def test_create_osti_label_xml(self): # Added/updated dates are always overwritten when parsing Doi objects # from input labels, so remove these key/values from the comparison - for date_key in ('date_record_added', 'date_record_updated'): + for date_key in ("date_record_added", "date_record_updated"): input_doi_fields.pop(date_key, None) output_doi_fields.pop(date_key, None) @@ -59,18 +55,14 @@ def test_create_osti_label_xml(self): def test_create_osti_label_json(self): """Test creation of an OSTI JSON label from Doi objects""" # Parse sample input to obtain a Doi object - input_json_file = join( - self.input_dir, 'DOI_Release_20210216_from_release.json' - ) + input_json_file = join(self.input_dir, "DOI_Release_20210216_from_release.json") - with open(input_json_file, 'r') as infile: + with open(input_json_file, "r") as infile: input_json = infile.read() dois, _ = DOIOstiJsonWebParser.parse_dois_from_label(input_json) # Now create an output label from the parsed Doi - output_json = DOIOstiRecord().create_doi_record( - dois[0], content_type=CONTENT_TYPE_JSON - ) + output_json = DOIOstiRecord().create_doi_record(dois[0], content_type=CONTENT_TYPE_JSON) # Massage the output a bit so we can do a straight dict comparison input_json = json.loads(input_json)[0] @@ -78,7 +70,7 @@ def test_create_osti_label_json(self): # Add/update dates are always overwritten when parsing Doi objects # from input labels, so remove these key/values from the comparison - for date_key in ('date_record_added', 'date_record_updated'): + for date_key in ("date_record_added", "date_record_updated"): input_json.pop(date_key, None) output_json.pop(date_key, None) @@ -90,25 +82,43 @@ class DOIOstiWebParserTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.test_dir = resource_filename(__name__, '') + cls.test_dir = resource_filename(__name__, "") # FIXME: moving this code from PACKAGE-DIR to PACKAGE-DIR/src (and, when we add # namepsace packages, to PACKAGE-DIR/src/pds) shouldn't necessitate re-jiggering # all the parent directories: - cls.input_dir = abspath( - join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'input') - ) - - cls.expected_authors = [{'first_name': 'R.', 'last_name': 'Deen'}, - {'first_name': 'H.', 'last_name': 'Abarca'}, - {'first_name': 'P.', 'last_name': 'Zamani'}, - {'first_name': 'J.', 'last_name': 'Maki'}] - cls.expected_editors = [{'first_name': 'P. H.', 'last_name': 'Smith'}, - {'first_name': 'M.', 'last_name': 'Lemmon'}, - {'first_name': 'R. F.', 'last_name': 'Beebe'}] - cls.expected_keywords = {'data', 'rdr', 'product', 'experiment', 'lander', - 'context', 'PDS', 'raw', 'mars', 'record', 'reduced', - 'science', 'edr', 'PDS4', 'camera', 'deployment', - 'insight', 'engineering'} + cls.input_dir = abspath(join(cls.test_dir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "input")) + + cls.expected_authors = [ + {"first_name": "R.", "last_name": "Deen"}, + {"first_name": "H.", "last_name": "Abarca"}, + {"first_name": "P.", "last_name": "Zamani"}, + {"first_name": "J.", "last_name": "Maki"}, + ] + cls.expected_editors = [ + {"first_name": "P. H.", "last_name": "Smith"}, + {"first_name": "M.", "last_name": "Lemmon"}, + {"first_name": "R. F.", "last_name": "Beebe"}, + ] + cls.expected_keywords = { + "data", + "rdr", + "product", + "experiment", + "lander", + "context", + "PDS", + "raw", + "mars", + "record", + "reduced", + "science", + "edr", + "PDS4", + "camera", + "deployment", + "insight", + "engineering", + } def _compare_doi_to_expected(self, doi): """ @@ -116,39 +126,36 @@ def _compare_doi_to_expected(self, doi): a parsed Doi match the expected values and/or formats. """ self.assertListEqual(doi.authors, self.expected_authors) - self.assertEqual(doi.availability, 'NASA Planetary Data System') - self.assertEqual(doi.contributor, 'Engineering') - self.assertEqual(doi.country, 'US') + self.assertEqual(doi.availability, "NASA Planetary Data System") + self.assertEqual(doi.contributor, "Engineering") + self.assertEqual(doi.country, "US") self.assertIsInstance(doi.date_record_added, datetime) - self.assertEqual(doi.description, - 'InSight Cameras Experiment Data Record (EDR) ' - 'and Reduced Data Record (RDR) Data Products') - self.assertEqual(doi.doi, '10.17189/29569') + self.assertEqual( + doi.description, + "InSight Cameras Experiment Data Record (EDR) " "and Reduced Data Record (RDR) Data Products", + ) + self.assertEqual(doi.doi, "10.17189/29569") self.assertListEqual(doi.editors, self.expected_editors) - self.assertEqual(doi.id, '29569') + self.assertEqual(doi.id, "29569") self.assertSetEqual(doi.keywords, self.expected_keywords) self.assertEqual(doi.product_type, ProductType.Dataset) - self.assertEqual(doi.product_type_specific, 'PDS4 Refereed Data Bundle') + self.assertEqual(doi.product_type_specific, "PDS4 Refereed Data Bundle") self.assertIsInstance(doi.publication_date, datetime) - self.assertEqual(doi.publisher, 'NASA Planetary Data System') - self.assertEqual(doi.related_identifier, - 'urn:nasa:pds:insight_cameras::1.0') + self.assertEqual(doi.publisher, "NASA Planetary Data System") + self.assertEqual(doi.related_identifier, "urn:nasa:pds:insight_cameras::1.0") # Check that site url HTML was un-escaped as expected - self.assertIn('&', doi.site_url) - self.assertNotIn('&', doi.site_url) - self.assertEqual(doi.sponsoring_organization, - 'National Aeronautics and Space Administration (NASA)') + self.assertIn("&", doi.site_url) + self.assertNotIn("&", doi.site_url) + self.assertEqual(doi.sponsoring_organization, "National Aeronautics and Space Administration (NASA)") self.assertEqual(doi.status, DoiStatus.Pending) - self.assertEqual(doi.title, 'InSight Cameras Bundle') + self.assertEqual(doi.title, "InSight Cameras Bundle") def test_parse_osti_response_xml(self): """Test parsing of an OSTI label in XML format""" # Test with a nominal file containing most of the optional fields - input_xml_file = join( - self.input_dir, 'DOI_Release_20200727_from_release.xml' - ) + input_xml_file = join(self.input_dir, "DOI_Release_20200727_from_release.xml") - with open(input_xml_file, 'r') as infile: + with open(input_xml_file, "r") as infile: input_xml = infile.read() dois, errors = DOIOstiXmlWebParser.parse_dois_from_label(input_xml) @@ -160,11 +167,9 @@ def test_parse_osti_response_xml(self): self._compare_doi_to_expected(doi) # Test with an erroneous file to ensure errors are parsed as we expect - input_xml_file = join( - self.input_dir, 'DOI_Release_20200727_from_error.xml' - ) + input_xml_file = join(self.input_dir, "DOI_Release_20200727_from_error.xml") - with open(input_xml_file, 'r') as infile: + with open(input_xml_file, "r") as infile: input_xml = infile.read() dois, errors = DOIOstiXmlWebParser.parse_dois_from_label(input_xml) @@ -174,11 +179,9 @@ def test_parse_osti_response_xml(self): def test_parse_osti_response_json(self): """Test parsing of an OSTI label in JSON format""" # Test with a nominal file containing most of the optional fields - input_json_file = join( - self.input_dir, 'DOI_Release_20210216_from_release.json' - ) + input_json_file = join(self.input_dir, "DOI_Release_20210216_from_release.json") - with open(input_json_file, 'r') as infile: + with open(input_json_file, "r") as infile: input_json = infile.read() dois, errors = DOIOstiJsonWebParser.parse_dois_from_label(input_json) @@ -190,11 +193,9 @@ def test_parse_osti_response_json(self): self._compare_doi_to_expected(doi) # Test with an erroneous file to ensure errors are parsed as we expect - input_json_file = join( - self.input_dir, 'DOI_Release_20210216_from_error.json' - ) + input_json_file = join(self.input_dir, "DOI_Release_20210216_from_error.json") - with open(input_json_file, 'r') as infile: + with open(input_json_file, "r") as infile: input_json = infile.read() dois, errors = DOIOstiJsonWebParser.parse_dois_from_label(input_json) @@ -202,5 +203,5 @@ def test_parse_osti_response_json(self): self.assertEqual(len(errors), 1) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/outputs/transaction.py b/src/pds_doi_service/core/outputs/transaction.py index 4357fe83..bfdcfef7 100644 --- a/src/pds_doi_service/core/outputs/transaction.py +++ b/src/pds_doi_service/core/outputs/transaction.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============== transaction.py @@ -13,7 +12,6 @@ Defines the Transaction class, which is used to log transactions both to local disk and database table. """ - from datetime import datetime from pds_doi_service.core.outputs.transaction_on_disk import TransactionOnDisk @@ -31,8 +29,9 @@ class Transaction: m_doi_config_util = DOIConfigUtil() - def __init__(self, output_content, output_content_type, node_id, - submitter_email, doi, transaction_db, input_path=None): + def __init__( + self, output_content, output_content_type, node_id, submitter_email, doi, transaction_db, input_path=None + ): self._config = self.m_doi_config_util.get_config() self._node_id = node_id.lower() self._submitter_email = submitter_email @@ -50,23 +49,25 @@ def output_content(self): def log(self): transaction_io_dir = self._transaction_disk.write( - self._node_id, self._transaction_time, input_ref=self._input_ref, + self._node_id, + self._transaction_time, + input_ref=self._input_ref, output_content=self._output_content, - output_content_type=self._output_content_type + output_content_type=self._output_content_type, ) doi_fields = self._doi.__dict__ self._transaction_db.write_doi_info_to_database( - identifier=doi_fields['related_identifier'], + identifier=doi_fields["related_identifier"], transaction_key=transaction_io_dir, - doi=doi_fields['doi'], - date_added=doi_fields.get('date_record_added', self._transaction_time), - date_updated=doi_fields.get('date_record_updated', self._transaction_time), - status=doi_fields['status'], - title=doi_fields['title'], - product_type=doi_fields['product_type'], - product_type_specific=doi_fields['product_type_specific'], + doi=doi_fields["doi"], + date_added=doi_fields.get("date_record_added", self._transaction_time), + date_updated=doi_fields.get("date_record_updated", self._transaction_time), + status=doi_fields["status"], + title=doi_fields["title"], + product_type=doi_fields["product_type"], + product_type_specific=doi_fields["product_type_specific"], submitter=self._submitter_email, - discipline_node=self._node_id + discipline_node=self._node_id, ) diff --git a/src/pds_doi_service/core/outputs/transaction_builder.py b/src/pds_doi_service/core/outputs/transaction_builder.py index 4f22c5ad..dda537be 100644 --- a/src/pds_doi_service/core/outputs/transaction_builder.py +++ b/src/pds_doi_service/core/outputs/transaction_builder.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ====================== transaction_builder.py @@ -13,12 +12,11 @@ Contains the TransactionBuilder class, which is used to manage transactions with the local database. """ - from pds_doi_service.core.db.doi_database import DOIDataBase -from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML, VALID_CONTENT_TYPES +from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML +from pds_doi_service.core.outputs.doi_record import VALID_CONTENT_TYPES from pds_doi_service.core.outputs.service import DOIServiceFactory from pds_doi_service.core.outputs.transaction import Transaction - from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger @@ -30,6 +28,7 @@ class TransactionBuilder: This class provides services to build a transaction, transaction logger, and database writer that can be used for writing to disk and/or to database. """ + m_doi_config_util = DOIConfigUtil() def __init__(self, db_name=None): @@ -38,12 +37,11 @@ def __init__(self, db_name=None): if db_name: self.m_doi_database = DOIDataBase(db_name) else: - self.m_doi_database = DOIDataBase(self._config.get('OTHER', 'db_file')) + self.m_doi_database = DOIDataBase(self._config.get("OTHER", "db_file")) self.record_service = DOIServiceFactory.get_doi_record_service() - def prepare_transaction(self, node_id, submitter_email, doi, input_path=None, - output_content_type=CONTENT_TYPE_XML): + def prepare_transaction(self, node_id, submitter_email, doi, input_path=None, output_content_type=CONTENT_TYPE_XML): """ Build a Transaction from the inputs and outputs to a 'reserve', 'draft' or release action. The Transaction object is returned. @@ -54,11 +52,10 @@ def prepare_transaction(self, node_id, submitter_email, doi, input_path=None, """ if output_content_type not in VALID_CONTENT_TYPES: - raise ValueError('Invalid content type requested, must be one of ' - f'{",".join(VALID_CONTENT_TYPES)}') + raise ValueError("Invalid content type requested, must be one of " f'{",".join(VALID_CONTENT_TYPES)}') # Get the latest available entry in the DB for this lidvid, if it exists - query_criteria = {'ids': [doi.related_identifier]} + query_criteria = {"ids": [doi.related_identifier]} columns, rows = self.m_doi_database.select_latest_rows(query_criteria) # Get the latest transaction record for this LIDVID so we can carry @@ -67,24 +64,24 @@ def prepare_transaction(self, node_id, submitter_email, doi, input_path=None, latest_row = dict(zip(columns, rows[0])) # Carry original release date forward - doi.date_record_added = latest_row['date_added'] + doi.date_record_added = latest_row["date_added"] # We might have a DOI already in the database from a previous reserve - if not doi.doi and latest_row['doi']: - doi.doi = latest_row['doi'] - doi.id = doi.doi.split('/')[-1] + if not doi.doi and latest_row["doi"]: + doi.doi = latest_row["doi"] + doi.id = doi.doi.split("/")[-1] # Create the output label that's written to the local transaction # history on disk. This label should represent the most up-to-date # version for this DOI/LIDVID - output_content = self.record_service.create_doi_record( - doi, content_type=output_content_type + output_content = self.record_service.create_doi_record(doi, content_type=output_content_type) + + return Transaction( + output_content, + output_content_type, + node_id, + submitter_email, + doi, + self.m_doi_database, + input_path=input_path, ) - - return Transaction(output_content, - output_content_type, - node_id, - submitter_email, - doi, - self.m_doi_database, - input_path=input_path) diff --git a/src/pds_doi_service/core/outputs/transaction_on_disk.py b/src/pds_doi_service/core/outputs/transaction_on_disk.py index bf592e8e..5be95400 100644 --- a/src/pds_doi_service/core/outputs/transaction_on_disk.py +++ b/src/pds_doi_service/core/outputs/transaction_on_disk.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ====================== transaction_on_disk.py @@ -13,18 +12,16 @@ Defines the TransactionOnDisk class, which manages writing of a transaction's input and output products to local disk. """ - import os -import requests import shutil - from distutils.dir_util import copy_tree +import requests from pds_doi_service.core.input.node_util import NodeUtil from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger -logger = get_logger('pds_doi_service.core.outputs.transaction_logger') +logger = get_logger("pds_doi_service.core.outputs.transaction_logger") class TransactionOnDisk: @@ -32,6 +29,7 @@ class TransactionOnDisk: This class provides services to write a transaction from an action (reserve, draft or release) to disk. """ + m_doi_config_util = DOIConfigUtil() m_node_util = NodeUtil() m_doi_database = None @@ -39,13 +37,12 @@ class TransactionOnDisk: def __init__(self): self._config = self.m_doi_config_util.get_config() - def write(self, node_id, update_time, input_ref=None, output_content=None, - output_content_type=None): + def write(self, node_id, update_time, input_ref=None, output_content=None, output_content_type=None): """ Write a the input and output products from a transaction to disk. The location of the written files is returned. """ - transaction_dir = self._config.get('OTHER', 'transaction_dir') + transaction_dir = self._config.get("OTHER", "transaction_dir") logger.debug(f"transaction_dir {transaction_dir}") # Create the local transaction history directory, if necessary. @@ -58,7 +55,7 @@ def write(self, node_id, update_time, input_ref=None, output_content=None, # Write input file with provided content. # Note that the file name is always 'input' plus the extension based # on the content_type (input.xml or input.csv or input.xlsx) - full_input_name = os.path.join(final_output_dir, 'input' + input_content_type) + full_input_name = os.path.join(final_output_dir, "input" + input_content_type) # If the provided content is actually a file name, we copy it, # otherwise write it to external file using full_input_name as name. @@ -69,7 +66,7 @@ def write(self, node_id, update_time, input_ref=None, output_content=None, else: # remote resource r = requests.get(input_ref, allow_redirects=True) - with open(full_input_name, 'wb') as outfile: + with open(full_input_name, "wb") as outfile: outfile.write(r.content) r.close() @@ -77,13 +74,11 @@ def write(self, node_id, update_time, input_ref=None, output_content=None, # Write output file with provided content # The extension of the file is determined by the provided content type if output_content and output_content_type: - full_output_name = os.path.join( - final_output_dir, '.'.join(['output', output_content_type]) - ) + full_output_name = os.path.join(final_output_dir, ".".join(["output", output_content_type])) - with open(full_output_name, 'w') as outfile: + with open(full_output_name, "w") as outfile: outfile.write(output_content) - logger.info(f'transaction files saved in {final_output_dir}') + logger.info(f"transaction files saved in {final_output_dir}") return final_output_dir diff --git a/src/pds_doi_service/core/outputs/web_client.py b/src/pds_doi_service/core/outputs/web_client.py index 7ea7aaad..db40097e 100644 --- a/src/pds_doi_service/core/outputs/web_client.py +++ b/src/pds_doi_service/core/outputs/web_client.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============= web_client.py @@ -13,34 +12,32 @@ Contains the abstract base class for interfacing with a DOI submission service endpoint. """ - -import pprint import json -import requests -from requests.auth import HTTPBasicAuth +import pprint +import requests from pds_doi_service.core.input.exceptions import WebRequestException from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML from pds_doi_service.core.util.config_parser import DOIConfigUtil +from requests.auth import HTTPBasicAuth -WEB_METHOD_GET = 'GET' -WEB_METHOD_POST = 'POST' -WEB_METHOD_PUT = 'PUT' -WEB_METHOD_DELETE = 'DELETE' -VALID_WEB_METHODS = [WEB_METHOD_GET, WEB_METHOD_POST, - WEB_METHOD_PUT, WEB_METHOD_DELETE] +WEB_METHOD_GET = "GET" +WEB_METHOD_POST = "POST" +WEB_METHOD_PUT = "PUT" +WEB_METHOD_DELETE = "DELETE" +VALID_WEB_METHODS = [WEB_METHOD_GET, WEB_METHOD_POST, WEB_METHOD_PUT, WEB_METHOD_DELETE] """Constants for HTTP method types""" class DOIWebClient: """Abstract base class for clients of an HTTP DOI service endpoint""" + _config_util = DOIConfigUtil() _service_name = None _web_parser = None _content_type_map = {} - def _submit_content(self, payload, url, username, password, - method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML): + def _submit_content(self, payload, url, username, password, method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML): """ Submits a payload to a DOI service endpoint via the POST action. @@ -72,43 +69,35 @@ def _submit_content(self, payload, url, username, password, """ if method not in VALID_WEB_METHODS: - raise ValueError('Invalid method requested, must be one of ' - f'{",".join(VALID_WEB_METHODS)}') + raise ValueError("Invalid method requested, must be one of " f'{",".join(VALID_WEB_METHODS)}') if content_type not in self._content_type_map: - raise ValueError('Invalid content type requested, must be one of ' - f'{",".join(list(self._content_type_map.keys()))}') + raise ValueError( + "Invalid content type requested, must be one of " f'{",".join(list(self._content_type_map.keys()))}' + ) auth = HTTPBasicAuth(username, password) - headers = { - 'Accept': self._content_type_map[content_type], - 'Content-Type': self._content_type_map[content_type] - } + headers = {"Accept": self._content_type_map[content_type], "Content-Type": self._content_type_map[content_type]} - response = requests.request( - method, url, auth=auth, data=payload, headers=headers - ) + response = requests.request(method, url, auth=auth, data=payload, headers=headers) try: response.raise_for_status() except requests.exceptions.HTTPError as http_err: # Detail text is not always present, which can cause json parsing # issues - details = ( - f'Details: {pprint.pformat(json.loads(response.text))}' - if response.text else '' - ) + details = f"Details: {pprint.pformat(json.loads(response.text))}" if response.text else "" raise WebRequestException( - f'DOI submission request to {self._service_name} service failed, ' - f'reason: {str(http_err)}\n{details}' + f"DOI submission request to {self._service_name} service failed, " f"reason: {str(http_err)}\n{details}" ) return response.text - def submit_content(self, payload, url=None, username=None, password=None, - method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML): + def submit_content( + self, payload, url=None, username=None, password=None, method=WEB_METHOD_POST, content_type=CONTENT_TYPE_XML + ): """ Submits the provided payload to a DOI service endpoint via the POST action. @@ -150,12 +139,10 @@ def submit_content(self, payload, url=None, username=None, password=None, """ raise NotImplementedError( - f'Subclasses of {self.__class__.__name__} must provide an ' - f'implementation for submit_content()' + f"Subclasses of {self.__class__.__name__} must provide an " f"implementation for submit_content()" ) - def query_doi(self, query, url=None, username=None, password=None, - content_type=CONTENT_TYPE_XML): + def query_doi(self, query, url=None, username=None, password=None, content_type=CONTENT_TYPE_XML): """ Queries the DOI endpoint for the status of a DOI submission. The query utilizes the GET HTTP method of the URL endpoint. @@ -190,6 +177,5 @@ def query_doi(self, query, url=None, username=None, password=None, """ raise NotImplementedError( - f'Subclasses of {self.__class__.__name__} must provide an ' - f'implementation for query_doi()' + f"Subclasses of {self.__class__.__name__} must provide an " f"implementation for query_doi()" ) diff --git a/src/pds_doi_service/core/outputs/web_parser.py b/src/pds_doi_service/core/outputs/web_parser.py index 170359b0..29340d7e 100644 --- a/src/pds_doi_service/core/outputs/web_parser.py +++ b/src/pds_doi_service/core/outputs/web_parser.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ============= web_parser.py @@ -13,7 +12,6 @@ Contains the abstract base class for parsing DOI objects from label returned or provided to DOI service endpoints (OSTI, Datacite, etc...). """ - from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_XML @@ -22,6 +20,7 @@ class DOIWebParser: Abstract base class for parsers of DOI labels returned (or submitted) to a DOI service endpoint. """ + _optional_fields = [] """The optional Doi field names parsed from labels.""" @@ -53,7 +52,7 @@ def _get_identifier_from_site_url(site_url): vid_value = lid_vid_tokens[1] # Finally combine the lid and vid together. - lid_vid_value = lid_value + '::' + vid_value + lid_vid_value = lid_value + "::" + vid_value return lid_vid_value @@ -79,8 +78,7 @@ def parse_dois_from_label(label_text, content_type=CONTENT_TYPE_XML): """ raise NotImplementedError( - f'Subclasses of {DOIWebParser.__name__} must provide an ' - f'implementation for parse_dois_from_label()' + f"Subclasses of {DOIWebParser.__name__} must provide an " f"implementation for parse_dois_from_label()" ) @staticmethod @@ -107,6 +105,5 @@ def get_record_for_identifier(label_file, identifier): """ raise NotImplementedError( - f'Subclasses of {DOIWebParser.__name__} must provide an ' - f'implementation for get_record_for_identifier()' + f"Subclasses of {DOIWebParser.__name__} must provide an " f"implementation for get_record_for_identifier()" ) diff --git a/src/pds_doi_service/core/references/contributors.py b/src/pds_doi_service/core/references/contributors.py index d28934b1..5d0f63dc 100644 --- a/src/pds_doi_service/core/references/contributors.py +++ b/src/pds_doi_service/core/references/contributors.py @@ -5,28 +5,28 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # -#------------------------------ - +# ------------------------------ import requests from pds_doi_service.core.util.general_util import get_logger # Get the common logger and set the level for this file. -logger = get_logger('pds_doi_core.references.contributor') +logger = get_logger("pds_doi_core.references.contributor") + class DOIContributorUtil: def __init__(self, dictionnary_url, pds_node_identifier): self._url = dictionnary_url self._pds_node_identifier = pds_node_identifier - def get_permissible_values(self): - '''Return a list of permissible values for contributors.''' - response = requests.get(self._url, stream=False, headers={'Connection': 'close'}) + """Return a list of permissible values for contributors.""" + response = requests.get(self._url, stream=False, headers={"Connection": "close"}) json_data = response.json() - attributeDictionaries = json_data[0]['dataDictionary']['attributeDictionary'] - node_dictionnary = [x for x in attributeDictionaries if x['attribute']['identifier']==self._pds_node_identifier][0]['attribute'] - - return [pv['PermissibleValue']['value'] for pv in node_dictionnary['PermissibleValueList']] + attributeDictionaries = json_data[0]["dataDictionary"]["attributeDictionary"] + node_dictionnary = [ + x for x in attributeDictionaries if x["attribute"]["identifier"] == self._pds_node_identifier + ][0]["attribute"] + return [pv["PermissibleValue"]["value"] for pv in node_dictionnary["PermissibleValueList"]] diff --git a/src/pds_doi_service/core/references/test/__init__.py b/src/pds_doi_service/core/references/test/__init__.py index a8c0b739..237250f9 100644 --- a/src/pds_doi_service/core/references/test/__init__.py +++ b/src/pds_doi_service/core/references/test/__init__.py @@ -1,11 +1,9 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for core references -''' - - +""" import unittest + from . import contributors_test diff --git a/src/pds_doi_service/core/references/test/contributors_test.py b/src/pds_doi_service/core/references/test/contributors_test.py index 9d47ae8e..5aa871db 100644 --- a/src/pds_doi_service/core/references/test/contributors_test.py +++ b/src/pds_doi_service/core/references/test/contributors_test.py @@ -1,25 +1,28 @@ import unittest + from pds_doi_service.core.references.contributors import DOIContributorUtil class MyTestCase(unittest.TestCase): - _doi_contributor_util = DOIContributorUtil('https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_JSON_1D00.JSON', - '0001_NASA_PDS_1.pds.Node.pds.name') + _doi_contributor_util = DOIContributorUtil( + "https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_JSON_1D00.JSON", "0001_NASA_PDS_1.pds.Node.pds.name" + ) def test_authorized_contributor(self): - - authorized_contributor = 'Cartography and Imaging Sciences Discipline' \ - in self._doi_contributor_util.get_permissible_values() + authorized_contributor = ( + "Cartography and Imaging Sciences Discipline" in self._doi_contributor_util.get_permissible_values() + ) self.assertEqual(authorized_contributor, True) def test_unauthorized_contributor(self): - authorized_contributor = 'Cartography and Imaging Sciences Disciine' \ - in self._doi_contributor_util.get_permissible_values() + authorized_contributor = ( + "Cartography and Imaging Sciences Disciine" in self._doi_contributor_util.get_permissible_values() + ) self.assertEqual(authorized_contributor, False) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/core/util/cmd_parser.py b/src/pds_doi_service/core/util/cmd_parser.py index 2556cab1..7554cbd0 100644 --- a/src/pds_doi_service/core/util/cmd_parser.py +++ b/src/pds_doi_service/core/util/cmd_parser.py @@ -1,23 +1,26 @@ - -def add_default_action_arguments(_parser,action_type): - _parser.add_argument('-c', '--node-id', - help='The pds discipline node in charge of the submission of the DOI', - required=True, - metavar='"img"') - _parser.add_argument('-i', '--input', - help='A pds4 label local or on http, a xls spreadsheet, a database file' - ' is also supported to reserve a list of doi', - required=True, - metavar='input/bundle_in_with_contributors.xml') - _parser.add_argument('-s', '--submitter-email', - help='The email address of the user performing the action for these services', - required=True, - metavar='"my.email@node.gov"') - _parser.add_argument('-t', '--target', - help='the system target to mint the DOI', - required=False, - default='osti', - metavar='osti') - - - +def add_default_action_arguments(_parser, action_type): + _parser.add_argument( + "-c", + "--node-id", + help="The pds discipline node in charge of the submission of the DOI", + required=True, + metavar='"img"', + ) + _parser.add_argument( + "-i", + "--input", + help="A pds4 label local or on http, a xls spreadsheet, a database file" + " is also supported to reserve a list of doi", + required=True, + metavar="input/bundle_in_with_contributors.xml", + ) + _parser.add_argument( + "-s", + "--submitter-email", + help="The email address of the user performing the action for these services", + required=True, + metavar='"my.email@node.gov"', + ) + _parser.add_argument( + "-t", "--target", help="the system target to mint the DOI", required=False, default="osti", metavar="osti" + ) diff --git a/src/pds_doi_service/core/util/config_parser.py b/src/pds_doi_service/core/util/config_parser.py index fd95eef9..fb0bbc75 100644 --- a/src/pds_doi_service/core/util/config_parser.py +++ b/src/pds_doi_service/core/util/config_parser.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ================ config_parser.py @@ -13,14 +12,15 @@ Classes and functions for locating and parsing the configuration file for the core DOI service. """ - import configparser +import functools import logging import os import sys +from os.path import abspath +from os.path import dirname +from os.path import join -import functools -from os.path import abspath, dirname, join from pkg_resources import resource_filename @@ -50,7 +50,7 @@ def get(self, section, option, *, raw=False, vars=None, fallback=object()): default ConfigParser.get() is returned. """ - env_var_key = '_'.join([section, option]).upper() + env_var_key = "_".join([section, option]).upper() if env_var_key in os.environ: return os.environ[env_var_key] @@ -59,13 +59,12 @@ def get(self, section, option, *, raw=False, vars=None, fallback=object()): class DOIConfigUtil: - @staticmethod def _resolve_relative_path(parser): # resolve relative path with sys.prefix base path for section in parser.sections(): for (key, val) in parser.items(section): - if key.endswith('_file') or key.endswith('_dir'): + if key.endswith("_file") or key.endswith("_dir"): parser[section][key] = os.path.abspath(os.path.join(sys.prefix, val)) return parser @@ -78,17 +77,15 @@ def get_config(): parser = DOIConfigParser() # default configuration - conf_default = 'conf.ini.default' + conf_default = "conf.ini.default" conf_default_path = resource_filename(__name__, conf_default) # user-specified configuration for production - conf_user = 'pds_doi_service.ini' + conf_user = "pds_doi_service.ini" conf_user_prod_path = os.path.join(sys.prefix, conf_user) # user-specified configuration for development - conf_user_dev_path = abspath( - join(dirname(__file__), os.pardir, os.pardir, os.pardir, conf_user) - ) + conf_user_dev_path = abspath(join(dirname(__file__), os.pardir, os.pardir, os.pardir, conf_user)) candidates_full_path = [conf_default_path, conf_user_prod_path, conf_user_dev_path] @@ -97,8 +94,8 @@ def get_config(): if not found: raise RuntimeError( - 'Could not find an INI configuration file to ' - f'parse from the following candidates: {candidates_full_path}' + "Could not find an INI configuration file to " + f"parse from the following candidates: {candidates_full_path}" ) # When providing multiple configs they are parsed in successive order, diff --git a/src/pds_doi_service/core/util/doi_xml_differ.py b/src/pds_doi_service/core/util/doi_xml_differ.py index 94ac3f36..b5ab8f89 100644 --- a/src/pds_doi_service/core/util/doi_xml_differ.py +++ b/src/pds_doi_service/core/util/doi_xml_differ.py @@ -5,16 +5,16 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # -#------------------------------ - +# ------------------------------ import datetime -from lxml import etree +from lxml import etree from pds_doi_service.core.input.exceptions import InputFormatException from pds_doi_service.core.input.node_util import NodeUtil from pds_doi_service.core.util.general_util import get_logger -logger = get_logger('pds_doi_core.util.doi_xml_differ') +logger = get_logger("pds_doi_core.util.doi_xml_differ") + class DOIDiffer: # This class provide a way to answer the question if two DOIs (XML format) are similar. @@ -27,51 +27,60 @@ def _resolve_special_fields(field_name, historical_value, new_value): # Historical may have keywords that will not be in new code so a list is needed to skip. # Use all lowercase for consistency. keywords_to_skip_compare = NodeUtil.get_permissible_values() - keywords_to_skip_compare.append('PDS3'.lower()) - + keywords_to_skip_compare.append("PDS3".lower()) o_difference_is_acceptable_flag = False - if field_name == 'id': + if field_name == "id": if new_value is None: o_difference_is_acceptable_flag = True # The DOI fromo 'draft' may not have the 'id' field. else: if historical_value.lstrip().rstrip() == new_value.lstrip().rstrip(): o_difference_is_acceptable_flag = True - elif field_name == 'date_record_added': + elif field_name == "date_record_added": o_difference_is_acceptable_flag = True # The record may be added on different day. - elif field_name == 'publication_date': + elif field_name == "publication_date": try: - reformatted_date = datetime.datetime.strptime(historical_value,"%Y-%m-%d") # Parse using yyyy-mm-dd format. - historical_date_as_string = reformatted_date.strftime('%m/%d/%Y') # Convert to dd/mm/yyyy format. + reformatted_date = datetime.datetime.strptime( + historical_value, "%Y-%m-%d" + ) # Parse using yyyy-mm-dd format. + historical_date_as_string = reformatted_date.strftime("%m/%d/%Y") # Convert to dd/mm/yyyy format. if historical_date_as_string == new_value: o_difference_is_acceptable_flag = True except Exception: logger.error(f"Failed to parse publication_date {historical_value} as %Y-%m-%d format.") - # Try a different method before raising + # Try a different method before raising try: - reformatted_date = datetime.datetime.strptime(historical_value,"%m/%d/%Y") # Parse using mm/dd/yyy format. - historical_date_as_string = reformatted_date.strftime('%Y-%m-%d') + reformatted_date = datetime.datetime.strptime( + historical_value, "%m/%d/%Y" + ) # Parse using mm/dd/yyy format. + historical_date_as_string = reformatted_date.strftime("%Y-%m-%d") if historical_date_as_string == new_value: o_difference_is_acceptable_flag = True except Exception: logger.error(f"Failed to parse publication_date {historical_value} as %m/%d/%Y format.") - raise InputFormatException(f"Failed to parse publication_date {historical_value} as %Y-%m-%d and %m/%d/%Y formats.") - elif field_name == 'keywords': + raise InputFormatException( + f"Failed to parse publication_date {historical_value} as %Y-%m-%d and %m/%d/%Y formats." + ) + elif field_name == "keywords": # We will relax the keywords check since there will be new module used to build the keywords. # Comment out the next two lines if desire to un-relax the check after the module has been added. - o_difference_is_acceptable_flag = True + o_difference_is_acceptable_flag = True return o_difference_is_acceptable_flag # Code is not pretty, will be clean up later. # The 'keywords' field can be in any particular order. Split the keywords using ';' and then perform the 'in' function. - historical_value_tokens = historical_value.split(';') - historical_value_tokens = [x.lstrip().rstrip().lower() for x in historical_value_tokens] # Remove leading and trailing blanks. - new_value_tokens = new_value.split(';') - new_value_tokens = [x.lstrip().rstrip().lower() for x in new_value_tokens] # Remove leading and trailing blanks. - o_difference_is_acceptable_flag = True + historical_value_tokens = historical_value.split(";") + historical_value_tokens = [ + x.lstrip().rstrip().lower() for x in historical_value_tokens + ] # Remove leading and trailing blanks. + new_value_tokens = new_value.split(";") + new_value_tokens = [ + x.lstrip().rstrip().lower() for x in new_value_tokens + ] # Remove leading and trailing blanks. + o_difference_is_acceptable_flag = True for ii in range(len(historical_value_tokens)): - # If one token differ, then the whole 'keywords' field is different. + # If one token differ, then the whole 'keywords' field is different. # Make a 2nd attempt by looking at each token in the list of tokens. if historical_value_tokens[ii].lower() in keywords_to_skip_compare: continue @@ -80,42 +89,50 @@ def _resolve_special_fields(field_name, historical_value, new_value): # If one token matches, then the difference is acceptable. o_difference_is_acceptable_flag = False for jj in range(len(new_value_tokens)): - if historical_value_tokens[ii] in new_value_tokens[jj]: + if historical_value_tokens[ii] in new_value_tokens[jj]: o_difference_is_acceptable_flag = True else: # Split the historical_value_tokens[ii] with spaces "solar wind" into ["solar", "wind"] space_split_historical_tokens = historical_value_tokens[ii].split() - logger.debug(f"KEYWORD_SEARCH_FAILED:len(space_split_historical_tokens {len(space_split_historical_tokens)}") + logger.debug( + f"KEYWORD_SEARCH_FAILED:len(space_split_historical_tokens {len(space_split_historical_tokens)}" + ) for kk in range(len(space_split_historical_tokens)): - logger.debug(f"KEYWORD_SEARCH_FAILED:INSPECTING_IN {space_split_historical_tokens[kk]} IN {new_value_tokens[jj],space_split_historical_tokens[kk] in new_value_tokens[jj]}") + logger.debug( + f"KEYWORD_SEARCH_FAILED:INSPECTING_IN {space_split_historical_tokens[kk]} IN {new_value_tokens[jj],space_split_historical_tokens[kk] in new_value_tokens[jj]}" + ) if space_split_historical_tokens[kk] in new_value_tokens[jj]: o_difference_is_acceptable_flag = True else: pass - logger.debug(f"KEYWORD_SEARCH_FAILED:historical_value_tokens[ii] NOT_IN new_value_tokens[jj] {historical_value_tokens[ii],new_value_tokens[jj]}") + logger.debug( + f"KEYWORD_SEARCH_FAILED:historical_value_tokens[ii] NOT_IN new_value_tokens[jj] {historical_value_tokens[ii],new_value_tokens[jj]}" + ) if not o_difference_is_acceptable_flag: - logger.error(f"KEYWORD_SEARCH_FAILED:[{historical_value_tokens[ii].lstrip().rstrip()} IN {new_value_tokens}") + logger.error( + f"KEYWORD_SEARCH_FAILED:[{historical_value_tokens[ii].lstrip().rstrip()} IN {new_value_tokens}" + ) logger.error(f"KEYWORD_SEARCH_FAILED:keywords_to_skip_compare [{keywords_to_skip_compare}") - #exit(0) - elif field_name == 'publisher': + # exit(0) + elif field_name == "publisher": # Historical value may be: "Atmospheres Node" # But new value may have "Planetary Data System:" preceding the node name as in "Planetary Data System: Atmospheres Node" # So an 'in' check may be sufficient. - if 'Node' in historical_value: - o_difference_is_acceptable_flag = True - elif field_name == 'availability': - if 'NSSDCA' in historical_value: - o_difference_is_acceptable_flag = True - elif field_name == 'product_type_specific': + if "Node" in historical_value: + o_difference_is_acceptable_flag = True + elif field_name == "availability": + if "NSSDCA" in historical_value: + o_difference_is_acceptable_flag = True + elif field_name == "product_type_specific": # Historical value may be: "PDS4 Bundle" # whereas new value may be "PDS4 Refereed Data Bundle" historical_value_tokens = historical_value.split() - o_difference_is_acceptable_flag = True + o_difference_is_acceptable_flag = True for ii in range(len(historical_value_tokens)): # If just one token in historical not in new, then the difference is not acceptable. if historical_value_tokens[ii].lower() not in new_value.lower(): o_difference_is_acceptable_flag = False - elif field_name == 'first_name': + elif field_name == "first_name": # Historical may not have the first_name as expected: # 'first_name': 'C. T.' # Whereas new code may have two distinct names. @@ -124,24 +141,26 @@ def _resolve_special_fields(field_name, historical_value, new_value): # so a check for in should be sufficient. if new_value.lower() in historical_value.lower(): o_difference_is_acceptable_flag = True - elif field_name == 'description' or field_name == 'title': + elif field_name == "description" or field_name == "title": # Remove tabs, new lines, spaces, split and join the string back to all lowercase. - sanitized_new_value = ' '.join(new_value.split()) - sanitized_new_value = [x.lower() for x in sanitized_new_value] - sanitized_historical_value = ' '.join(historical_value.split()) - sanitized_historical_value = [x.lower() for x in sanitized_historical_value] + sanitized_new_value = " ".join(new_value.split()) + sanitized_new_value = [x.lower() for x in sanitized_new_value] + sanitized_historical_value = " ".join(historical_value.split()) + sanitized_historical_value = [x.lower() for x in sanitized_historical_value] if sanitized_new_value == sanitized_historical_value: o_difference_is_acceptable_flag = True - elif field_name == 'full_name': + elif field_name == "full_name": # ['historical:[PDS Geosciences (GEO) Node], new:[Planetary Data System: Geosciences Node]'] historical_value_tokens = historical_value.split() - keywords_to_skip_compare.append('pds') + keywords_to_skip_compare.append("pds") new_list_to_skip_compare = [] - for keyword in keywords_to_skip_compare: + for keyword in keywords_to_skip_compare: new_list_to_skip_compare.append(keyword) - new_list_to_skip_compare.append('(' + keyword + ')') # Add the parenthesis to each keyword so it become ('geo') - sanitized_new_value = new_value.split() - sanitized_new_value = [x.lower() for x in sanitized_new_value] + new_list_to_skip_compare.append( + "(" + keyword + ")" + ) # Add the parenthesis to each keyword so it become ('geo') + sanitized_new_value = new_value.split() + sanitized_new_value = [x.lower() for x in sanitized_new_value] num_tokens_compared = 0 for historical_token in historical_value_tokens: @@ -151,14 +170,17 @@ def _resolve_special_fields(field_name, historical_value, new_value): continue if historical_token.lower() in sanitized_new_value: num_tokens_compared += 1 - logger.debug(f"FULL_NAME:historical_token,sanitized_new_value,o_difference_is_acceptable_flag {historical_token,sanitized_new_value,o_difference_is_acceptable_flag}") + logger.debug( + f"FULL_NAME:historical_token,sanitized_new_value,o_difference_is_acceptable_flag {historical_token,sanitized_new_value,o_difference_is_acceptable_flag}" + ) - logger.debug(f"FULL_NAME:historical_value,sanitized_new_value,o_difference_is_acceptable_flag {historical_value,sanitized_new_value,o_difference_is_acceptable_flag,new_list_to_skip_compare,num_tokens_compared,len(historical_value_tokens)}") + logger.debug( + f"FULL_NAME:historical_value,sanitized_new_value,o_difference_is_acceptable_flag {historical_value,sanitized_new_value,o_difference_is_acceptable_flag,new_list_to_skip_compare,num_tokens_compared,len(historical_value_tokens)}" + ) # If all tokens from historical_value can be found in new_value then we are successful. if num_tokens_compared == len(historical_value_tokens): o_difference_is_acceptable_flag = True -# exit(0) - + # exit(0) return o_difference_is_acceptable_flag @@ -169,7 +191,7 @@ def _resolve_apparent_differences(field_name, historical_value, new_value): # It is possible that the value may differ depending on how the software was written. For example: # # Historical element: - # + # # URL # urn:nasa:pds:bundle_mpf_asimet::1.0 # Cites @@ -186,19 +208,23 @@ def _resolve_apparent_differences(field_name, historical_value, new_value): if not field_name: return o_difference_is_acceptable_flag - _acceptable_values_dict = {'identifier_type': ['URL','URN'], - 'relation_type' : ['Cites','IsIdenticalTo'], - 'product_type' : ['Collection','Bundle', 'Dataset', 'Text']} + _acceptable_values_dict = { + "identifier_type": ["URL", "URN"], + "relation_type": ["Cites", "IsIdenticalTo"], + "product_type": ["Collection", "Bundle", "Dataset", "Text"], + } # If the given field name is one of the keys in _acceptable_values_dict, look to see if the value is acceptable. if field_name in _acceptable_values_dict: # Check to see if each value in the acceptable values also occurs in historical_value - for ii, acceptable_value in enumerate(_acceptable_values_dict[field_name]): + for ii, acceptable_value in enumerate(_acceptable_values_dict[field_name]): if acceptable_value.lower() in historical_value.lower(): o_difference_is_acceptable_flag = True - break; + break else: - logger.debug(f"Cannot find field_name {field_name} in _acceptable_values_dict. Will call _resolve_special_fields()") + logger.debug( + f"Cannot find field_name {field_name} in _acceptable_values_dict. Will call _resolve_special_fields()" + ) # Make another attempting by checking to see if some fields have different format or values. o_difference_is_acceptable_flag = DOIDiffer._resolve_special_fields(field_name, historical_value, new_value) @@ -211,40 +237,50 @@ def _determine_xpath_tag(child_level_1): # 'first_name' in 'authors/author/first_name' xpath or # 'last_name' in 'authors/author/last_name' xpath. - o_xpath_tag = child_level_1.tag + o_xpath_tag = child_level_1.tag original_tag = child_level_1.tag # Travel up from child_level_1.tag until no more parent and build the xpath. levels_travelled_up = 0 if child_level_1.getparent() is not None: - levels_travelled_up += 1 # LEVEL 1 + levels_travelled_up += 1 # LEVEL 1 if child_level_1.getparent().getparent() is not None: levels_travelled_up += 1 # LEVEL 2 - o_xpath_tag = original_tag + o_xpath_tag = original_tag if child_level_1.getparent().getparent().getparent() is not None: - levels_travelled_up += 1 # LEVEL 3 - o_xpath_tag = child_level_1.getparent().getparent().tag + '/' + child_level_1.getparent().tag + '/' + original_tag + levels_travelled_up += 1 # LEVEL 3 + o_xpath_tag = ( + child_level_1.getparent().getparent().tag + + "/" + + child_level_1.getparent().tag + + "/" + + original_tag + ) else: logger.debug(f"child_level_1.getparent().getparent() is indeed None {child_level_1.tag}") - if child_level_1.tag == 'accession_number': - o_xpath_tag = 'product_nos' # The new code uses 'product_nos' tag instead of 'accession_number' - if child_level_1.tag == 'date_record_added': + if child_level_1.tag == "accession_number": + o_xpath_tag = "product_nos" # The new code uses 'product_nos' tag instead of 'accession_number' + if child_level_1.tag == "date_record_added": pass logger.debug(f"FINAL_TAG {o_xpath_tag,original_tag,levels_travelled_up}") return o_xpath_tag - def _compare_individual_field(historical_element,child_level_1,new_element,element_index,indices_where_field_occur_dict): + def _compare_individual_field( + historical_element, child_level_1, new_element, element_index, indices_where_field_occur_dict + ): # Given an XML element from a historical document, compare the field in the element with the new document. o_field_differ_name = None # Returning the name of the field that is not similiar. - o_values_differ = None # Returning a string containing the historical value and the new value to allow the user to inspect. + o_values_differ = ( + None # Returning a string containing the historical value and the new value to allow the user to inspect. + ) historical_has_children = False if len(child_level_1): - historical_has_children = True + historical_has_children = True # It is possible that the element to compare comes from a closing tag, which means there is no .text field. # Which means there is no meaningful work to perform. @@ -253,45 +289,58 @@ def _compare_individual_field(historical_element,child_level_1,new_element,eleme if not historical_has_children: o_xpath_tag = DOIDiffer._determine_xpath_tag(child_level_1) - # Find the element in the new element with the same same tag as the historical element. - new_child = new_element.xpath(o_xpath_tag) + # Find the element in the new element with the same same tag as the historical element. + new_child = new_element.xpath(o_xpath_tag) if new_child is None or len(new_child) == 0: - logger.warning(f"New code does not produced field {child_level_1.tag} in DOI output. Will skip comparing this field.") + logger.warning( + f"New code does not produced field {child_level_1.tag} in DOI output. Will skip comparing this field." + ) return o_field_differ_name, o_values_differ - - field_index = DOIDiffer._get_indices_where_tag_occur(child_level_1.tag,child_level_1.getparent().tag,indices_where_field_occur_dict) + field_index = DOIDiffer._get_indices_where_tag_occur( + child_level_1.tag, child_level_1.getparent().tag, indices_where_field_occur_dict + ) # Because some fields are forced to exist eventhough it is an empty string, check for None-ness otherwise # the lstrip() will failed on None. e.g logger.info(f"child_level_1.tag,field_index,len(new_child) {child_level_1.tag,field_index,len(new_child)}") if new_child[field_index].text is not None: - if (child_level_1.text.lstrip().rstrip() == new_child[field_index].text.lstrip().rstrip()): - pass # Field is the same which is good. - logger.debug(f"FIELD_SAME_TRUE: {child_level_1,child_level_1.text} == {new_child[field_index].text}") + if child_level_1.text.lstrip().rstrip() == new_child[field_index].text.lstrip().rstrip(): + pass # Field is the same which is good. + logger.debug( + f"FIELD_SAME_TRUE: {child_level_1,child_level_1.text} == {new_child[field_index].text}" + ) else: # Fields are different. Attempt to resolve the apparent differences. - logger.debug(f"FIELD_SAME_FALSE: {child_level_1,child_level_1.text} != {new_child[field_index].text}") + logger.debug( + f"FIELD_SAME_FALSE: {child_level_1,child_level_1.text} != {new_child[field_index].text}" + ) # It is possible for new_child[0].text to be None so do not perform lstrip() and rstrip() - o_difference_is_acceptable_flag = DOIDiffer._resolve_apparent_differences(field_name=child_level_1.tag, historical_value=child_level_1.text, new_value=new_child[0].text) + o_difference_is_acceptable_flag = DOIDiffer._resolve_apparent_differences( + field_name=child_level_1.tag, historical_value=child_level_1.text, new_value=new_child[0].text + ) if not o_difference_is_acceptable_flag: # The differences are not acceptable between historical and new field, save the field name. o_field_differ_name = child_level_1.tag # Save the values different. o_values_differ = "historical:[" + child_level_1.text + "], new:[" + new_child[0].text + "]" - logger.info(f"FIELD_SAME_FALSE_FINALLY: {child_level_1,child_level_1.text} != {new_child[field_index].text}") + logger.info( + f"FIELD_SAME_FALSE_FINALLY: {child_level_1,child_level_1.text} != {new_child[field_index].text}" + ) else: - logger.info(f"FIELD_SAME_TRUE_FINALLY: {child_level_1,child_level_1.text} == {new_child[field_index].text}") + logger.info( + f"FIELD_SAME_TRUE_FINALLY: {child_level_1,child_level_1.text} == {new_child[field_index].text}" + ) return o_field_differ_name, o_values_differ def _pre_condition_documents(historical_doc, new_doc): # Pre-condition both documents before comparing so they have the same order. - new_root = new_doc.getroot() + new_root = new_doc.getroot() historical_root = historical_doc.getroot() identifier_list_from_historical = [] @@ -306,7 +355,9 @@ def _pre_condition_documents(historical_doc, new_doc): sorting_element = element.xpath("related_identifiers/related_identifier/identifier_value") else: sorting_element = element.xpath("accession_number") - logger.info(f":related_identifiers/related_identifier/identifier_value:len(sorting_element) {len(sorting_element)}") + logger.info( + f":related_identifiers/related_identifier/identifier_value:len(sorting_element) {len(sorting_element)}" + ) if len(sorting_element) == 0: sorting_element = element.xpath("product_nos") logger.info(f":product_nos:len(sorting_element) {len(sorting_element)}") @@ -339,25 +390,25 @@ def _pre_condition_documents(historical_doc, new_doc): new_root.append(new_dict_list[key]) # Re-parse both documents now with the 'record' elements in the same order. - new_doc = etree.fromstring(etree.tostring(new_root)) + new_doc = etree.fromstring(etree.tostring(new_root)) historical_doc = etree.fromstring(etree.tostring(historical_root)) return historical_doc, new_doc - def _update_indices_where_tag_occur(tag_name,my_parent_tag,indices_where_field_occur_dict): + def _update_indices_where_tag_occur(tag_name, my_parent_tag, indices_where_field_occur_dict): # Check if 'first_name' of 'author' is in dictionary or not and increment by 1 if found. - full_tag = my_parent_tag + '_'+ tag_name + full_tag = my_parent_tag + "_" + tag_name if full_tag in indices_where_field_occur_dict: - indices_where_field_occur_dict[full_tag] += 1 + indices_where_field_occur_dict[full_tag] += 1 else: pass return indices_where_field_occur_dict - def _get_indices_where_tag_occur(tag_name,my_parent_tag,indices_where_field_occur_dict): + def _get_indices_where_tag_occur(tag_name, my_parent_tag, indices_where_field_occur_dict): # Return where in the tree where the combination of my_parent_tag and tag_name occur, e.g. # author_first_name # contributor_contributor_type - full_tag = my_parent_tag + '_'+ tag_name + full_tag = my_parent_tag + "_" + tag_name if full_tag in indices_where_field_occur_dict: return indices_where_field_occur_dict[full_tag] else: @@ -365,17 +416,26 @@ def _get_indices_where_tag_occur(tag_name,my_parent_tag,indices_where_field_occu return 0 def _setup_where_field_occur_dict(): - # For fields that can have multiple occurences, a dictionary is necessary to + # For fields that can have multiple occurences, a dictionary is necessary to # remember where each field occur in the historical tree so it can be used to find the field in the new tree. - indices_where_field_occur_dict = {'author_first_name' : 0, - 'author_last_name' : 0, - 'contributor_first_name' : 0, - 'contributor_last_name' : 0, - 'contributor_full_name' : 0, - 'contributor_contributor_type' : 0} + indices_where_field_occur_dict = { + "author_first_name": 0, + "author_last_name": 0, + "contributor_first_name": 0, + "contributor_last_name": 0, + "contributor_full_name": 0, + "contributor_contributor_type": 0, + } return indices_where_field_occur_dict - def _differ_single_record(new_doc,historical_element,element_index,io_fields_differ_list,io_values_differ_list,io_record_index_differ_list): + def _differ_single_record( + new_doc, + historical_element, + element_index, + io_fields_differ_list, + io_values_differ_list, + io_record_index_differ_list, + ): # Given a 'record' element, compare all fields within and return info about any fields that differed. indices_where_field_occur_dict = DOIDiffer._setup_where_field_occur_dict() @@ -391,27 +451,29 @@ def _differ_single_record(new_doc,historical_element,element_index,io_fields_dif for child_level_1 in historical_element.iter(): # The element with the tag 'record' is not useful, so it is skipped. # The Comment element is also not useful, so it is skipped. - if child_level_1.tag == 'record' or child_level_1.tag is etree.Comment: + if child_level_1.tag == "record" or child_level_1.tag is etree.Comment: continue # Do the fields compare and save the field name differ and the two values list. - o_field_differ_name, o_values_differ = DOIDiffer._compare_individual_field(historical_element,child_level_1,new_element,element_index,indices_where_field_occur_dict) + o_field_differ_name, o_values_differ = DOIDiffer._compare_individual_field( + historical_element, child_level_1, new_element, element_index, indices_where_field_occur_dict + ) my_parent_tag = child_level_1.getparent().tag - DOIDiffer._update_indices_where_tag_occur(child_level_1.tag,my_parent_tag,indices_where_field_occur_dict) + DOIDiffer._update_indices_where_tag_occur(child_level_1.tag, my_parent_tag, indices_where_field_occur_dict) if o_field_differ_name: io_fields_differ_list.append(o_field_differ_name) # Save the field name that differs. - io_values_differ_list.append(o_values_differ) # Save the values where the fields differ. + io_values_differ_list.append(o_values_differ) # Save the values where the fields differ. io_record_index_differ_list.append(element_index) # Save the index where the fields differ. child_index += 1 # end for child_level_1 in historical_element.iter(): - return io_fields_differ_list,io_values_differ_list,io_record_index_differ_list + return io_fields_differ_list, io_values_differ_list, io_record_index_differ_list @staticmethod - def doi_xml_differ(historical_xml_output,new_xml_output): + def doi_xml_differ(historical_xml_output, new_xml_output): # Function compares two XML file specifically the output from a 'reserve' or 'draft' action. # Assumptions: # 1. The elements in the XML tree may not share the same order, so they will be sorted by title. @@ -421,7 +483,7 @@ def doi_xml_differ(historical_xml_output,new_xml_output): # # - # + # # Apollo 15 and 17 Heat Flow Experiment Concatenated Data Sets Bundle this title is different # # @@ -432,7 +494,7 @@ def doi_xml_differ(historical_xml_output,new_xml_output): # 02/27/2020 # Collection is different # PDS4 Bundle - # + # # # # URL instead of URN @@ -445,7 +507,7 @@ def doi_xml_differ(historical_xml_output,new_xml_output): o_record_index_differ_list = [] # A list of indices where the fields differ. # Build an XML tree from both input. - new_doc = etree.parse(new_xml_output) + new_doc = etree.parse(new_xml_output) historical_doc = etree.parse(historical_xml_output) # Because the ordering of the 'record' element in the document tree is not known, @@ -462,12 +524,23 @@ def doi_xml_differ(historical_xml_output,new_xml_output): records_compared = 0 for historical_element in historical_doc.iter(): - if historical_element.tag == 'record': + if historical_element.tag == "record": # Special logic: # Sometimes historical code does not produce the correct number of records. One example # was historical only produced 6 and new code produces 8. - logger.debug(f'elemet index {element_index}') - o_fields_differ_list,o_values_differ_list,o_record_index_differ_list = DOIDiffer._differ_single_record(new_doc,historical_element,element_index,o_fields_differ_list,o_values_differ_list,o_record_index_differ_list) + logger.debug(f"elemet index {element_index}") + ( + o_fields_differ_list, + o_values_differ_list, + o_record_index_differ_list, + ) = DOIDiffer._differ_single_record( + new_doc, + historical_element, + element_index, + o_fields_differ_list, + o_values_differ_list, + o_record_index_differ_list, + ) element_index += 1 records_compared += 1 @@ -476,22 +549,25 @@ def doi_xml_differ(historical_xml_output,new_xml_output): return o_fields_differ_list, o_values_differ_list, o_record_index_differ_list -if __name__ == '__main__': - historical_xml_output = '/Users/loubrieu/PycharmProjects/pds-doi-service/aaDOI_production_submitted_labels/PPI_InSight_Bundles_Collections_20200812/aaRegistered_by_EN_active/DOI_registered_all_records_corrected.xml' - new_xml_output = '/Users/loubrieu/PycharmProjects/pds-doi-service/output/test.xml' - #historical_xml_output = os.path.join("./","aaaSubmitted_by_ATMOS_reserve_2020624","DOI_reserved_all_records.xml") - #new_xml_output = 'DOI_Reserve_ATM-2020-06-30_from_new_code.xml' +if __name__ == "__main__": + historical_xml_output = "/Users/loubrieu/PycharmProjects/pds-doi-service/aaDOI_production_submitted_labels/PPI_InSight_Bundles_Collections_20200812/aaRegistered_by_EN_active/DOI_registered_all_records_corrected.xml" + new_xml_output = "/Users/loubrieu/PycharmProjects/pds-doi-service/output/test.xml" + + # historical_xml_output = os.path.join("./","aaaSubmitted_by_ATMOS_reserve_2020624","DOI_reserved_all_records.xml") + # new_xml_output = 'DOI_Reserve_ATM-2020-06-30_from_new_code.xml' - #historical_xml_output = 'temp_doi_label.xml' - #new_xml_output = 'temp_doi_label.xml' + # historical_xml_output = 'temp_doi_label.xml' + # new_xml_output = 'temp_doi_label.xml' - #historical_xml_output = 'aaDOI_production_submitted_labels/GEO_Insight_cruise_20200611/aaRegistered_by_EN/DOI_registered_all_records.xml' - #new_xml_output = 'temp_doi_label.xml' + # historical_xml_output = 'aaDOI_production_submitted_labels/GEO_Insight_cruise_20200611/aaRegistered_by_EN/DOI_registered_all_records.xml' + # new_xml_output = 'temp_doi_label.xml' - o_fields_differ_list, o_values_differ_list, o_record_index_differ_list = DOIDiffer.doi_xml_differ(historical_xml_output,new_xml_output) + o_fields_differ_list, o_values_differ_list, o_record_index_differ_list = DOIDiffer.doi_xml_differ( + historical_xml_output, new_xml_output + ) - print("o_fields_differ_list",len(o_fields_differ_list),o_fields_differ_list) - print("o_values_differ_list",len(o_values_differ_list),o_values_differ_list) - print("o_record_index_differ_list",len(o_record_index_differ_list),o_record_index_differ_list) - print("historical_xml_output,new_xml_output",historical_xml_output,new_xml_output) + print("o_fields_differ_list", len(o_fields_differ_list), o_fields_differ_list) + print("o_values_differ_list", len(o_values_differ_list), o_values_differ_list) + print("o_record_index_differ_list", len(o_record_index_differ_list), o_record_index_differ_list) + print("historical_xml_output,new_xml_output", historical_xml_output, new_xml_output) diff --git a/src/pds_doi_service/core/util/emailer.py b/src/pds_doi_service/core/util/emailer.py index 18d756be..6580a6b5 100644 --- a/src/pds_doi_service/core/util/emailer.py +++ b/src/pds_doi_service/core/util/emailer.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ ========== emailer.py @@ -12,13 +11,12 @@ Utilities for sending email messages. """ - import smtplib from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger -logger = get_logger('pds_doi_service.core.util.emailer') +logger = get_logger("pds_doi_service.core.util.emailer") class Emailer: @@ -29,8 +27,8 @@ class Emailer: def __init__(self): self._config = DOIConfigUtil().get_config() - self.m_localhost = self._config.get('OTHER', 'emailer_local_host') - self.m_email_port = int(self._config.get('OTHER', 'emailer_port')) + self.m_localhost = self._config.get("OTHER", "emailer_local_host") + self.m_email_port = int(self._config.get("OTHER", "emailer_port")) def sendmail(self, sender, receivers, subject, message_body): """ @@ -51,10 +49,7 @@ def sendmail(self, sender, receivers, subject, message_body): # Build the output message using all parameters. # Note that this format below allows the email recipient to see the # subject, sender, recipient(s) clearly - out_message = (f"From: {sender}\n" - f"To: {','.join(receivers)}\n" - f"Subject: {subject}\n\n" - f"{message_body}") + out_message = f"From: {sender}\n" f"To: {','.join(receivers)}\n" f"Subject: {subject}\n\n" f"{message_body}" smtp = None diff --git a/src/pds_doi_service/core/util/general_util.py b/src/pds_doi_service/core/util/general_util.py index 04ae3040..29a2ebeb 100644 --- a/src/pds_doi_service/core/util/general_util.py +++ b/src/pds_doi_service/core/util/general_util.py @@ -4,7 +4,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ =============== general_util.py @@ -12,9 +11,8 @@ General utility functions for things like logging. """ - -import re import logging +import re from pds_doi_service.core.util.config_parser import DOIConfigUtil @@ -41,7 +39,7 @@ def sanitize_json_string(string): return re.sub(r"\s+", " ", string, flags=re.UNICODE).strip() -def get_logger(module_name=''): +def get_logger(module_name=""): # If the user specifies the module name, we can use it. if module_name: logger = logging.getLogger(module_name) @@ -50,12 +48,10 @@ def get_logger(module_name=''): my_format = "%(levelname)s %(name)s:%(funcName)s %(message)s" - logging.basicConfig(format=my_format, filemode='a') + logging.basicConfig(format=my_format, filemode="a") config = DOIConfigUtil().get_config() - logging_level = config.get('OTHER', 'logging_level') + logging_level = config.get("OTHER", "logging_level") logger.setLevel(getattr(logging, logging_level.upper())) return logger - - diff --git a/src/pds_doi_service/core/util/initialize_production_deployment.py b/src/pds_doi_service/core/util/initialize_production_deployment.py index 27e4ae5f..6aa04c28 100644 --- a/src/pds_doi_service/core/util/initialize_production_deployment.py +++ b/src/pds_doi_service/core/util/initialize_production_deployment.py @@ -5,7 +5,6 @@ # use must be negotiated with the Office of Technology Transfer at the # California Institute of Technology. # - """ =================================== initialize_production_deployment.py @@ -14,7 +13,6 @@ Script used to import the available DOIs from a service provider into the local production database. """ - # Parameters to this script: # # The -S (optional) is the name of the DOI service provider to pull existing @@ -44,7 +42,6 @@ # initialize_production_deployment.py -s pds-operator@jpl.nasa.gov -d temp.db --dry-run --debug >& t1 ; tail -20 t1 # initialize_production_deployment.py -s pds-operator@jpl.nasa.gov -i my_input.xml -d temp.db --debug >& t1 ; tail -20 t1 # initialize_production_deployment.py -s pds-operator@jpl.nasa.gov -d temp.db --debug >& t1 ; tail -20 t1 - # Note: As of 10/01/2020, there is one DOI (10.17189/1517674) that does not have # any info for lidvid that caused the software to crash. There may be more. # @@ -60,20 +57,19 @@ # Ron has been notified. # Note: As of 10/06/2020, there are 1058 DOIs in the OPS OSTI server associated # with the NASA-PDS account. - import argparse import json import logging import os from datetime import datetime -from pds_doi_service.core.input.exceptions import (InputFormatException, - CriticalDOIException) -from pds_doi_service.core.outputs.service import (DOIServiceFactory, - SERVICE_TYPE_DATACITE, - VALID_SERVICE_TYPES) -from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiXmlWebParser +from pds_doi_service.core.input.exceptions import CriticalDOIException +from pds_doi_service.core.input.exceptions import InputFormatException from pds_doi_service.core.outputs.doi_record import CONTENT_TYPE_JSON +from pds_doi_service.core.outputs.osti.osti_web_parser import DOIOstiXmlWebParser +from pds_doi_service.core.outputs.service import DOIServiceFactory +from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE +from pds_doi_service.core.outputs.service import VALID_SERVICE_TYPES from pds_doi_service.core.outputs.transaction_builder import TransactionBuilder from pds_doi_service.core.util.config_parser import DOIConfigUtil from pds_doi_service.core.util.general_util import get_logger @@ -88,52 +84,77 @@ def create_cmd_parser(): parser = argparse.ArgumentParser( - description='Script to bulk import existing DOIs into the local ' - 'transaction database.', - epilog='Note: When DOI records are imported to the local transaction ' - 'database, the DOI service creates an associated output label ' - 'for each record under the transaction_history directory. The ' - 'format of this output label is driven by the SERVICE.provider ' - 'field of the INI. Please ensure the field is set appropriately ' - 'before using this script, as a mismatch could cause parsing ' - 'errors when using the DOI service after this script.' + description="Script to bulk import existing DOIs into the local " "transaction database.", + epilog="Note: When DOI records are imported to the local transaction " + "database, the DOI service creates an associated output label " + "for each record under the transaction_history directory. The " + "format of this output label is driven by the SERVICE.provider " + "field of the INI. Please ensure the field is set appropriately " + "before using this script, as a mismatch could cause parsing " + "errors when using the DOI service after this script.", + ) + parser.add_argument( + "-S", + "--service", + required=False, + default=None, + help="Name of the service provider to pull existing DOI " + "records from. If not provided, the provider configured " + "by the DOI service configuration INI is used by " + "default. Should be one of: [{}]".format(", ".join(VALID_SERVICE_TYPES)), + ) + parser.add_argument( + "-p", + "--prefix", + required=False, + default=None, + help="Specify the DOI prefix value to query the service " + "provider for. If not provided, the prefix value " + "configured to the providing in the INI config is " + "used by default.", + ) + parser.add_argument( + "-s", + "--submitter-email", + required=False, + default="pds-operator@jpl.nasa.gov", + help="The email address of the user performing the " + "deployment database initialization. Defaults to " + "pds-operator@jpl.nasa.gov.", ) - parser.add_argument("-S", "--service", required=False, default=None, - help="Name of the service provider to pull existing DOI " - "records from. If not provided, the provider configured " - "by the DOI service configuration INI is used by " - "default. Should be one of: [{}]" - .format(", ".join(VALID_SERVICE_TYPES))) - parser.add_argument("-p", "--prefix", required=False, default=None, - help="Specify the DOI prefix value to query the service " - "provider for. If not provided, the prefix value " - "configured to the providing in the INI config is " - "used by default.") - parser.add_argument("-s", "--submitter-email", required=False, - default='pds-operator@jpl.nasa.gov', - help="The email address of the user performing the " - "deployment database initialization. Defaults to " - "pds-operator@jpl.nasa.gov.") - parser.add_argument("-d", "--db-name", required=False, - help="Name of the SQLite3 database file name to commit " - "DOI records to. If not provided, the file name is " - "obtained from the DOI service INI config.") - parser.add_argument("-i", "--input-file", required=False, - help="Input file (XML or JSON) to import existing DOIs from. " - "If no value is provided, the server URL " - "specified by the DOI service configuration INI " - "file is used by default.") - parser.add_argument("-o", "--output-file", required=False, default=None, - help="Path to write out the DOI JSON labels as returned " - "from the query. When created, this file can be used " - "with the --input option to import records at a " - "later time without re-querying the server. " - "This option has no effect if --input already " - "specifies an input file.") - parser.add_argument("--dry-run", required=False, action="store_true", - help="Flag to suppress actual writing of DOIs to database.") - parser.add_argument("--debug", required=False, action="store_true", - help="Flag to print debug statements.") + parser.add_argument( + "-d", + "--db-name", + required=False, + help="Name of the SQLite3 database file name to commit " + "DOI records to. If not provided, the file name is " + "obtained from the DOI service INI config.", + ) + parser.add_argument( + "-i", + "--input-file", + required=False, + help="Input file (XML or JSON) to import existing DOIs from. " + "If no value is provided, the server URL " + "specified by the DOI service configuration INI " + "file is used by default.", + ) + parser.add_argument( + "-o", + "--output-file", + required=False, + default=None, + help="Path to write out the DOI JSON labels as returned " + "from the query. When created, this file can be used " + "with the --input option to import records at a " + "later time without re-querying the server. " + "This option has no effect if --input already " + "specifies an input file.", + ) + parser.add_argument( + "--dry-run", required=False, action="store_true", help="Flag to suppress actual writing of DOIs to database." + ) + parser.add_argument("--debug", required=False, action="store_true", help="Flag to print debug statements.") return parser @@ -157,7 +178,7 @@ def _read_from_local_xml(path): """ try: - with open(path, mode='r') as f: + with open(path, mode="r") as f: doi_xml = f.read() except Exception as e: raise CriticalDOIException(str(e)) @@ -189,7 +210,7 @@ def _read_from_local_json(service, path): """ try: - with open(path, mode='r') as f: + with open(path, mode="r") as f: doi_json = f.read() except Exception as e: raise CriticalDOIException(str(e)) @@ -197,9 +218,7 @@ def _read_from_local_json(service, path): web_parser = DOIServiceFactory.get_web_parser_service(service) try: - dois, _ = web_parser.parse_dois_from_label( - doi_json, content_type=CONTENT_TYPE_JSON - ) + dois, _ = web_parser.parse_dois_from_label(doi_json, content_type=CONTENT_TYPE_JSON) except Exception: raise InputFormatException( f"Unable to parse input file {path} using parser {web_parser.__name__}\n" @@ -237,16 +256,14 @@ def _read_from_path(service, path): """ if not os.path.exists(path): - raise InputFormatException(f"Error reading file {path}. " - "File may not exist.") + raise InputFormatException(f"Error reading file {path}. " "File may not exist.") - if path.endswith('.xml'): + if path.endswith(".xml"): return _read_from_local_xml(path) - elif path.endswith('.json'): + elif path.endswith(".json"): return _read_from_local_json(service, path) - raise InputFormatException(f'File {path} is not supported. ' - f'Only .xml and .json are supported.') + raise InputFormatException(f"File {path} is not supported. " f"Only .xml and .json are supported.") def get_dois_from_provider(service, prefix, output_file=None): @@ -273,31 +290,27 @@ def get_dois_from_provider(service, prefix, output_file=None): """ if service == SERVICE_TYPE_DATACITE: - query_dict = {'doi': f'{prefix}/*'} + query_dict = {"doi": f"{prefix}/*"} else: - query_dict = {'doi': prefix} + query_dict = {"doi": prefix} - server_url = m_config.get(service.upper(), 'url') + server_url = m_config.get(service.upper(), "url") logger.info("Using %s server URL %s", service, server_url) web_client = DOIServiceFactory.get_web_client_service(service) - doi_json = web_client.query_doi( - query=query_dict, content_type=CONTENT_TYPE_JSON - ) + doi_json = web_client.query_doi(query=query_dict, content_type=CONTENT_TYPE_JSON) if output_file: logger.info("Writing query results to %s", output_file) - with open(output_file, 'w') as outfile: + with open(output_file, "w") as outfile: json.dump(json.loads(doi_json), outfile, indent=4) web_parser = DOIServiceFactory.get_web_parser_service(service) - dois, _ = web_parser.parse_dois_from_label( - doi_json, content_type=CONTENT_TYPE_JSON - ) + dois, _ = web_parser.parse_dois_from_label(doi_json, content_type=CONTENT_TYPE_JSON) return dois, server_url @@ -320,47 +333,46 @@ def _get_node_id_from_contributors(doi_fields): field. """ - node_id = 'eng' + node_id = "eng" - if doi_fields.get('contributor'): - full_name_orig = doi_fields['contributor'] + if doi_fields.get("contributor"): + full_name_orig = doi_fields["contributor"] full_name = full_name_orig.lower() - if 'atmospheres' in full_name: - node_id = 'atm' - elif 'engineering' in full_name: - node_id = 'eng' - elif 'geosciences' in full_name: - node_id = 'geo' - elif 'imaging' in full_name: - node_id = 'img' - elif 'cartography' in full_name: - node_id = 'img' + if "atmospheres" in full_name: + node_id = "atm" + elif "engineering" in full_name: + node_id = "eng" + elif "geosciences" in full_name: + node_id = "geo" + elif "imaging" in full_name: + node_id = "img" + elif "cartography" in full_name: + node_id = "img" # Some uses title: Navigation and Ancillary Information Facility Node # Some uses title: Navigational and Ancillary Information Facility # So check for both - elif 'navigation' in full_name and 'ancillary' in full_name: - node_id = 'naif' - elif 'navigational' in full_name and 'ancillary' in full_name: - node_id = 'naif' - elif 'plasma' in full_name: - node_id = 'ppi' - elif 'ring' in full_name and 'moon' in full_name: - node_id = 'rms' - elif 'small' in full_name or 'bodies' in full_name: - node_id = 'sbn' - - logger.debug("Derived node ID %s from Contributor field %s", - node_id, full_name_orig) + elif "navigation" in full_name and "ancillary" in full_name: + node_id = "naif" + elif "navigational" in full_name and "ancillary" in full_name: + node_id = "naif" + elif "plasma" in full_name: + node_id = "ppi" + elif "ring" in full_name and "moon" in full_name: + node_id = "rms" + elif "small" in full_name or "bodies" in full_name: + node_id = "sbn" + + logger.debug("Derived node ID %s from Contributor field %s", node_id, full_name_orig) else: - logger.warning("No Contributor field available for DOI %s, " - "defaulting to node ID %s", doi_fields['doi'], node_id) + logger.warning( + "No Contributor field available for DOI %s, " "defaulting to node ID %s", doi_fields["doi"], node_id + ) return node_id -def perform_import_to_database(service, prefix, db_name, input_source, dry_run, - submitter_email, output_file): +def perform_import_to_database(service, prefix, db_name, input_source, dry_run, submitter_email, output_file): """ Imports all records from the input source into a local database. The input source may either be an existing file containing DOIs to parse, @@ -402,14 +414,14 @@ def perform_import_to_database(service, prefix, db_name, input_source, dry_run, logger.info("Using source service provider %s", service) if not prefix: - prefix = m_config.get(service.upper(), 'doi_prefix') + prefix = m_config.get(service.upper(), "doi_prefix") logger.info("Using DOI prefix %s", prefix) # If db_name is not provided, get one from config file: if not db_name: # This is the local database we'll be writing to - db_name = m_config.get('OTHER', 'db_file') + db_name = m_config.get("OTHER", "db_file") logger.info("Using local database %s", db_name) @@ -434,8 +446,7 @@ def perform_import_to_database(service, prefix, db_name, input_source, dry_run, # If the field 'related_identifier' is None, we cannot proceed since # it serves as the primary key for our transaction database. if not doi.related_identifier: - logger.warning("Skipping DOI with missing related identifier %s, " - "index %d", doi.doi, item_index) + logger.warning("Skipping DOI with missing related identifier %s, " "index %d", doi.doi, item_index) o_records_dois_skipped += 1 continue @@ -446,12 +457,12 @@ def perform_import_to_database(service, prefix, db_name, input_source, dry_run, node_id = _get_node_id_from_contributors(doi_fields) logger.debug("------------------------------------") - logger.debug('Processed DOI at index %d', item_index) - logger.debug("Title: %s", doi_fields.get('title')) - logger.debug("DOI: %s", doi_fields.get('doi')) - logger.debug("Related Identifier: %s", doi_fields.get('related_identifier')) + logger.debug("Processed DOI at index %d", item_index) + logger.debug("Title: %s", doi_fields.get("title")) + logger.debug("DOI: %s", doi_fields.get("doi")) + logger.debug("Related Identifier: %s", doi_fields.get("related_identifier")) logger.debug("Node ID: %s", node_id) - logger.debug("Status: %s", str(doi_fields.get('status', 'unknown'))) + logger.debug("Status: %s", str(doi_fields.get("status", "unknown"))) o_records_processed += 1 @@ -461,18 +472,14 @@ def perform_import_to_database(service, prefix, db_name, input_source, dry_run, # of the output label is based on the service provider setting in # the INI config. transaction = transaction_builder.prepare_transaction( - node_id, - submitter_email, - doi, - output_content_type=CONTENT_TYPE_JSON + node_id, submitter_email, doi, output_content_type=CONTENT_TYPE_JSON ) transaction.log() o_records_written += 1 - return (o_records_found, o_records_processed, o_records_written, - o_records_dois_skipped) + return (o_records_found, o_records_processed, o_records_written, o_records_dois_skipped) def main(): @@ -489,20 +496,19 @@ def main(): if arguments.debug: logger.setLevel(logging.DEBUG) - logger.info('Starting DOI import to local database...') - logger.debug('Command-line args: %r', arguments) + logger.info("Starting DOI import to local database...") + logger.debug("Command-line args: %r", arguments) # Do the import operation from remote server to database. - (records_found, - records_processed, - records_written, - records_skipped) = perform_import_to_database(arguments.service, - arguments.prefix, - arguments.db_name, - arguments.input_file, - arguments.dry_run, - arguments.submitter_email, - arguments.output_file) + (records_found, records_processed, records_written, records_skipped) = perform_import_to_database( + arguments.service, + arguments.prefix, + arguments.db_name, + arguments.input_file, + arguments.dry_run, + arguments.submitter_email, + arguments.output_file, + ) stop_time = datetime.now() elapsed_seconds = stop_time.timestamp() - start_time.timestamp() @@ -514,5 +520,5 @@ def main(): logger.info("Num records skipped: %d", records_skipped) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/pds_doi_service/core/util/keyword_tokenizer.py b/src/pds_doi_service/core/util/keyword_tokenizer.py index 98324e91..aeb2429c 100644 --- a/src/pds_doi_service/core/util/keyword_tokenizer.py +++ b/src/pds_doi_service/core/util/keyword_tokenizer.py @@ -1,30 +1,31 @@ import re + import nltk -nltk.download('stopwords', quiet=True) +nltk.download("stopwords", quiet=True) from nltk.corpus import stopwords -nltk.download('wordnet', quiet=True) +nltk.download("wordnet", quiet=True) from nltk.stem.wordnet import WordNetLemmatizer from pds_doi_service.core.util.general_util import get_logger logger = get_logger(__name__) -class KeywordTokenizer(): +class KeywordTokenizer: def __init__(self): - logger.debug('initialize keyword tokenizer') - self.pos_dict = {'pds', 'mars'} + logger.debug("initialize keyword tokenizer") + self.pos_dict = {"pds", "mars"} self._stop_words = set(stopwords.words("english")) self._keywords = set() def process_text(self, text): # Remove punctuations - logger.debug(f'extract keywords from {text}') - text = re.sub('^[^a-zA-Z0-9]+', ' ', text) - text = re.sub('[^a-zA-Z0-9]+$', ' ', text) - text = re.sub('[^a-zA-Z0-9]+ ', ' ', text) - text = re.sub(' [^a-zA-Z0-9]+', ' ', text) + logger.debug(f"extract keywords from {text}") + text = re.sub("^[^a-zA-Z0-9]+", " ", text) + text = re.sub("[^a-zA-Z0-9]+$", " ", text) + text = re.sub("[^a-zA-Z0-9]+ ", " ", text) + text = re.sub(" [^a-zA-Z0-9]+", " ", text) # Convert to lowercase text = text.lower() @@ -40,12 +41,13 @@ def process_text(self, text): # Lemmatisation lem = WordNetLemmatizer() - keyword_set = set([word if word in self.pos_dict else lem.lemmatize(word) - for word in text if word not in self._stop_words]) + keyword_set = set( + [word if word in self.pos_dict else lem.lemmatize(word) for word in text if word not in self._stop_words] + ) self._keywords |= keyword_set - logger.debug(f'new keyword list is {self._keywords}') + logger.debug(f"new keyword list is {self._keywords}") def get_keywords(self): return self._keywords diff --git a/src/pds_doi_service/core/util/test/__init__.py b/src/pds_doi_service/core/util/test/__init__.py index c5b4fc67..fdc903a7 100644 --- a/src/pds_doi_service/core/util/test/__init__.py +++ b/src/pds_doi_service/core/util/test/__init__.py @@ -1,11 +1,9 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests for core utilities -''' - - +""" import unittest + from . import config_parser_test diff --git a/src/pds_doi_service/core/util/test/config_parser_test.py b/src/pds_doi_service/core/util/test/config_parser_test.py index 4212b33d..bd1b6aa3 100644 --- a/src/pds_doi_service/core/util/test/config_parser_test.py +++ b/src/pds_doi_service/core/util/test/config_parser_test.py @@ -1,16 +1,14 @@ #!/usr/bin/env python - -import unittest -import sys import os +import sys +import unittest +from pds_doi_service.core.util.config_parser import DOIConfigParser +from pds_doi_service.core.util.config_parser import DOIConfigUtil from pkg_resources import resource_filename -from pds_doi_service.core.util.config_parser import DOIConfigParser, DOIConfigUtil - class ConfigParserTest(unittest.TestCase): - def test_add_absolute_path(self): """ Test that DOIConfigUtil._resolve_relative_path() prepends the @@ -18,10 +16,8 @@ def test_add_absolute_path(self): """ parser = DOIConfigUtil().get_config() - self.assertEqual(parser['OTHER']['db_file'], - os.path.join(sys.prefix, 'doi.db')) - self.assertEqual(parser['OTHER']['transaction_dir'], - os.path.join(sys.prefix, 'transaction_history')) + self.assertEqual(parser["OTHER"]["db_file"], os.path.join(sys.prefix, "doi.db")) + self.assertEqual(parser["OTHER"]["transaction_dir"], os.path.join(sys.prefix, "transaction_history")) def test_doi_config_parser(self): """ @@ -30,36 +26,37 @@ def test_doi_config_parser(self): parser = DOIConfigParser() # Populate our config parser with the default INI - conf_file_path = resource_filename('pds_doi_service', 'core/util/conf.ini.default') + conf_file_path = resource_filename("pds_doi_service", "core/util/conf.ini.default") parser.read(conf_file_path) # Ensure we get values from the default INI to begin with - self.assertEqual(parser['OSTI']['user'], 'username') - self.assertEqual(parser['PDS4_DICTIONARY']['pds_node_identifier'], - '0001_NASA_PDS_1.pds.Node.pds.name') - self.assertEqual(parser['LANDING_PAGES']['url'], - 'https://pds.nasa.gov/ds-view/pds/view{}.jsp?identifier={}&version={}') - self.assertEqual(parser['OTHER']['db_file'], 'doi.db') + self.assertEqual(parser["OSTI"]["user"], "username") + self.assertEqual(parser["PDS4_DICTIONARY"]["pds_node_identifier"], "0001_NASA_PDS_1.pds.Node.pds.name") + self.assertEqual( + parser["LANDING_PAGES"]["url"], "https://pds.nasa.gov/ds-view/pds/view{}.jsp?identifier={}&version={}" + ) + self.assertEqual(parser["OTHER"]["db_file"], "doi.db") # Now provide some environment variables to override with - os.environ['OSTI_USER'] = 'actual_username' - os.environ['PDS4_DICTIONARY_PDS_NODE_IDENTIFIER'] = '123ABC' - os.environ['LANDING_PAGES_URL'] = 'https://zombo.com' - os.environ['OTHER_DB_FILE'] = '/path/to/other/doi.db' + os.environ["OSTI_USER"] = "actual_username" + os.environ["PDS4_DICTIONARY_PDS_NODE_IDENTIFIER"] = "123ABC" + os.environ["LANDING_PAGES_URL"] = "https://zombo.com" + os.environ["OTHER_DB_FILE"] = "/path/to/other/doi.db" # Our config parser should prioritize the environment variables try: - self.assertEqual(parser['OSTI']['user'], os.environ['OSTI_USER']) - self.assertEqual(parser['PDS4_DICTIONARY']['pds_node_identifier'], - os.environ['PDS4_DICTIONARY_PDS_NODE_IDENTIFIER']) - self.assertEqual(parser['LANDING_PAGES']['url'], os.environ['LANDING_PAGES_URL']) - self.assertEqual(parser['OTHER']['db_file'], os.environ['OTHER_DB_FILE']) + self.assertEqual(parser["OSTI"]["user"], os.environ["OSTI_USER"]) + self.assertEqual( + parser["PDS4_DICTIONARY"]["pds_node_identifier"], os.environ["PDS4_DICTIONARY_PDS_NODE_IDENTIFIER"] + ) + self.assertEqual(parser["LANDING_PAGES"]["url"], os.environ["LANDING_PAGES_URL"]) + self.assertEqual(parser["OTHER"]["db_file"], os.environ["OTHER_DB_FILE"]) finally: - os.environ.pop('OSTI_USER') - os.environ.pop('PDS4_DICTIONARY_PDS_NODE_IDENTIFIER') - os.environ.pop('LANDING_PAGES_URL') - os.environ.pop('OTHER_DB_FILE') + os.environ.pop("OSTI_USER") + os.environ.pop("PDS4_DICTIONARY_PDS_NODE_IDENTIFIER") + os.environ.pop("LANDING_PAGES_URL") + os.environ.pop("OTHER_DB_FILE") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/pds_doi_service/test.py b/src/pds_doi_service/test.py index e59b05b0..13a9e446 100644 --- a/src/pds_doi_service/test.py +++ b/src/pds_doi_service/test.py @@ -1,18 +1,16 @@ # encoding: utf-8 - -''' +""" Planetary Data System's Digital Object Identifier service — tests. -''' - - +""" import unittest -import pds_doi_service.core.util.test -import pds_doi_service.core.input.test -import pds_doi_service.core.references.test + +import pds_doi_service.api.test import pds_doi_service.core.actions.test import pds_doi_service.core.db.test +import pds_doi_service.core.input.test import pds_doi_service.core.outputs.test -import pds_doi_service.api.test +import pds_doi_service.core.references.test +import pds_doi_service.core.util.test def suite(): diff --git a/tests/data/valid_browsecoll_doi.xml b/tests/data/valid_browsecoll_doi.xml index c73f5ec1..2b4a014f 100644 --- a/tests/data/valid_browsecoll_doi.xml +++ b/tests/data/valid_browsecoll_doi.xml @@ -46,4 +46,3 @@ 818.393.7165 - diff --git a/tests/data/valid_bundle_doi.xml b/tests/data/valid_bundle_doi.xml index f737bad7..ac985521 100644 --- a/tests/data/valid_bundle_doi.xml +++ b/tests/data/valid_bundle_doi.xml @@ -46,4 +46,3 @@ 818.393.7165 - diff --git a/tests/data/valid_calibcoll_doi.xml b/tests/data/valid_calibcoll_doi.xml index 7703fede..ca145535 100644 --- a/tests/data/valid_calibcoll_doi.xml +++ b/tests/data/valid_calibcoll_doi.xml @@ -46,4 +46,3 @@ 818.393.7165 - diff --git a/tests/data/valid_datacoll_doi.xml b/tests/data/valid_datacoll_doi.xml index 1e2a277d..86ab6c6b 100644 --- a/tests/data/valid_datacoll_doi.xml +++ b/tests/data/valid_datacoll_doi.xml @@ -46,4 +46,3 @@ 818.393.7165 - diff --git a/tests/data/valid_docucoll_doi.xml b/tests/data/valid_docucoll_doi.xml index 24428e77..a08ea4c6 100644 --- a/tests/data/valid_docucoll_doi.xml +++ b/tests/data/valid_docucoll_doi.xml @@ -46,4 +46,3 @@ 818.393.7165 - diff --git a/tests/end_to_end/reserve.csv b/tests/end_to_end/reserve.csv index 34cfafeb..03c41ce8 100644 --- a/tests/end_to_end/reserve.csv +++ b/tests/end_to_end/reserve.csv @@ -1,2 +1,2 @@ status,title,publication_date,product_type_specific,author_last_name,author_first_name,related_resource -Reserved,Laboratory Shocked Feldspars Collection #1,2020-03-11,PDS4 Collection,Johnson,J. R.,{{random_lid}}::1.0 \ No newline at end of file +Reserved,Laboratory Shocked Feldspars Collection #1,2020-03-11,PDS4 Collection,Johnson,J. R.,{{random_lid}}::1.0 diff --git a/tests/reserve_ok/output.xml b/tests/reserve_ok/output.xml index 2e28219b..1d783a29 100644 --- a/tests/reserve_ok/output.xml +++ b/tests/reserve_ok/output.xml @@ -90,4 +90,4 @@ pds-operator@jpl.nasa.gov 818.393.7165 - \ No newline at end of file + diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..cc84bd4d --- /dev/null +++ b/tox.ini @@ -0,0 +1,23 @@ +[tox] +envlist = py39, docs, lint + +[testenv] +deps = .[dev] +whitelist_externals = pytest +commands = pytest + +[testenv:docs] +deps = .[dev] +whitelist_externals = python +commands = python setup.py build_sphinx + +[testenv:lint] +deps = pre-commit +commands= + python -m pre_commit run --color=always {posargs:--all} +skip_install = true + +[testenv:dev] +basepython = python3.9 +usedevelop = True +deps = .[dev]