More progress towards lint free code maybe

.gitignore

@@ -59,6 +59,7 @@ var/
 *.xpr
 bin/
 .*.swp
+typescript
 
 # OS-specific artifacts
 .DS_Store

pyproject.toml

@@ -1,9 +1,13 @@
 [build-system]
-requires = ["setuptools >= 46.4.0", "wheel"]
+requires = ["setuptools >= 46.4.0", "wheel", "versioneer-518"]
 
 # uncomment to enable pep517 after versioneer problem is fixed.
 # https://github.com/python-versioneer/python-versioneer/issues/193
 #build-backend = "setuptools.build_meta"
 
 [tool.black]
 line-length = 120
+
+[tool.pytest.ini_options]
+# This is required for non-re-entrant tests in ``pds-doi-service``:
+numprocesses = 1

setup.cfg

@@ -91,6 +91,16 @@ dev =
     behave==1.2.6
     allure-behave==2.8.13
     behave-testrail-reporter==0.4.0
+    versioneer
+    types-python-dateutil
+    types-pkg_resources
+    lxml-stubs
+    pandas-stubs
+    types-requests
+    types-six
+    types-jsonschema
+    types-flask
+    types-waitress
 
 
 [options.entry_points]
@@ -127,7 +137,16 @@ docstring_convention = google
 # E501 prevents flake8 from complaining line lengths > 79. We will use
 # flake8-bugbear's B950 to handle line length lint errors. This trips
 # when a line is > max-line-length + 10%.
-extend-ignore = E203, E501
+#
+# W503 is ignored for ``pds-doi-service`` not just because it makes use of
+# left-aligned binary operators in multi-line expressions, but because the
+# opposite advice is now the recommended practice; see
+# • https://rhodesmill.org/brandon/slides/2012-11-pyconca/#laying-down-the-law
+# • https://github.com/PyCQA/pycodestyle/pull/502
+# • https://www.flake8rules.com/rules/W503.html
+# • ET Tufte, _Seeing with Fresh Eyes: Meaning, Space, Data, Truth_, Graphics
+#   Press 2020, p.14.
+extend-ignore = E203, E501, W503
 
 # Selects following test categories:
 # D: Docstring errors and warnings
@@ -147,7 +166,7 @@ select = D,E,F,N,W,B,B902,B903,B950
 
 [mypy]
 
-[mypy-pds.*._version]
+[mypy-pds_doi_service._version]
 # We don't care about issues in versioneer's files
 ignore_errors = True
 

src/pds_doi_service/api/__main__.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.
 #
 import logging
 from urllib.parse import urlparse
 
-import connexion
+import connexion  # type: ignore
 from flask import jsonify
-from flask_cors import CORS
+from flask_cors import CORS  # type: ignore
 from pds_doi_service.api import encoder
 from pds_doi_service.core.util.config_parser import DOIConfigUtil
 from pds_doi_service.core.util.general_util import get_logger

src/pds_doi_service/api/controllers/dois_controller.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.
@@ -18,7 +18,7 @@
 from os.path import join
 from tempfile import NamedTemporaryFile
 
-import connexion
+import connexion  # type: ignore
 from flask import current_app
 from pds_doi_service.api.models import DoiRecord
 from pds_doi_service.api.models import DoiSummary

src/pds_doi_service/api/encoder.py

@@ -1,5 +1,5 @@
 import six
-from connexion.apps.flask_app import FlaskJSONEncoder
+from connexion.apps.flask_app import FlaskJSONEncoder  # type: ignore
 from pds_doi_service.api.models import Model
 
 

src/pds_doi_service/api/models/base_model_.py

@@ -8,12 +8,14 @@
 
 
 class Model(object):
-    # swaggerTypes: The key is attribute name and the
-    # value is attribute type.
+    # swaggerTypes: The key is attribute name and the value is attribute type.
+    # 🤔 TODO: figure out the actual types that should go here
+    swagger_types: dict[typing.Any, typing.Any]
     swagger_types = {}
 
-    # attributeMap: The key is attribute name and the
-    # value is json key in definition.
+    # attributeMap: The key is attribute name and the value is json key in definition.
+    # 🤔 TODO: figure out the actual types that should go here
+    attribute_map: dict[typing.Any, typing.Any]
     attribute_map = {}
 
     @classmethod

src/pds_doi_service/api/models/label_payload.py

@@ -1,10 +1,7 @@
 # coding: utf-8
 from __future__ import absolute_import
 
-from datetime import date
 from datetime import datetime
-from typing import Dict
-from typing import List
 
 from pds_doi_service.api import util
 from pds_doi_service.api.models import Model

src/pds_doi_service/api/models/labels_payload.py

@@ -1,9 +1,6 @@
 # coding: utf-8
 from __future__ import absolute_import
 
-from datetime import date
-from datetime import datetime
-from typing import Dict
 from typing import List
 
 from pds_doi_service.api import util

src/pds_doi_service/api/test/_base.py

@@ -4,7 +4,7 @@
 """
 import logging
 
-from flask_testing import TestCase
+from flask_testing import TestCase  # type: ignore
 from pds_doi_service.api.__main__ import init_app
 
 

src/pds_doi_service/api/test/test_dois_controller.py

@@ -699,7 +699,7 @@ def test_get_doi_from_id(self):
         query_string = [("identifier", "urn:nasa:pds:insight_cameras::1.1")]
 
         response = self.client.open(
-            "/PDS_APIs/pds_doi_api/0.2/doi".format(lidvid="urn:nasa:pds:insight_cameras::1.1"),
+            "/PDS_APIs/pds_doi_api/0.2/doi",
             method="GET",
             query_string=query_string,
             headers={"Referer": "http://localhost"},

src/pds_doi_service/core/actions/__init__.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21 by the California Institute of Technology. ALL RIGHTS
+#  Copyright 2021–21 by the California Institute of Technology. ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.
@@ -12,10 +12,10 @@
 This package contains the implementations for the user-facing action classes
 used to interact with the PDS DOI service.
 """
-from pds_doi_service.core.actions.action import create_parser
-from pds_doi_service.core.actions.action import DOICoreAction
-from pds_doi_service.core.actions.check import DOICoreActionCheck
-from pds_doi_service.core.actions.draft import DOICoreActionDraft
-from pds_doi_service.core.actions.list import DOICoreActionList
-from pds_doi_service.core.actions.release import DOICoreActionRelease
-from pds_doi_service.core.actions.reserve import DOICoreActionReserve
+from pds_doi_service.core.actions.action import create_parser  # noqa: F401
+from pds_doi_service.core.actions.action import DOICoreAction  # noqa: F401
+from pds_doi_service.core.actions.check import DOICoreActionCheck  # noqa: F401
+from pds_doi_service.core.actions.draft import DOICoreActionDraft  # noqa: F401
+from pds_doi_service.core.actions.list import DOICoreActionList  # noqa: F401
+from pds_doi_service.core.actions.release import DOICoreActionRelease  # noqa: F401
+from pds_doi_service.core.actions.reserve import DOICoreActionReserve  # noqa: F401

src/pds_doi_service/core/actions/action.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21 by the California Institute of Technology. ALL RIGHTS
+#  Copyright 2021–21 by the California Institute of Technology. ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.
@@ -32,6 +32,7 @@ class DOICoreAction:
     _name = "unknown"
     _description = "no description"
     _order = 9999999  # used to sort actions in documentation
+    _run_arguments: tuple[str, ...]
     _run_arguments = ()
 
     def __init__(self, db_name=None):

src/pds_doi_service/core/actions/check.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.
@@ -20,7 +20,7 @@
 from email.mime.multipart import MIMEMultipart
 from os.path import exists
 
-import pystache
+import pystache  # type: ignore
 from pds_doi_service.core.actions import DOICoreAction
 from pds_doi_service.core.actions.list import DOICoreActionList
 from pds_doi_service.core.entities.doi import DoiStatus
@@ -414,7 +414,7 @@ def run(self, **kwargs):
         o_doi_list = self._list_obj.run(status=DoiStatus.Pending)
         pending_state_list = json.loads(o_doi_list)
 
-        logger.info(f"Found %d %s record(s) to check", len(pending_state_list), DoiStatus.Pending)
+        logger.info("Found %d %s record(s) to check" % (len(pending_state_list), DoiStatus.Pending))
 
         if len(pending_state_list) > 0:
             for pending_record in pending_state_list:

src/pds_doi_service/core/actions/draft.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.

src/pds_doi_service/core/actions/list.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.

src/pds_doi_service/core/actions/release.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.

src/pds_doi_service/core/actions/reserve.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.

src/pds_doi_service/core/cmd/pds_doi_cmd.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.

src/pds_doi_service/core/db/doi_database.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.
@@ -12,7 +12,6 @@
 Contains classes and functions for interfacing with the local transaction
 database (SQLite3).
 """
-import datetime
 import sqlite3
 from collections import OrderedDict
 from datetime import datetime

src/pds_doi_service/core/entities/doi.py

@@ -16,6 +16,8 @@
 from datetime import datetime
 from enum import Enum
 from enum import unique
+from typing import Any
+from typing import Optional
 
 
 @unique
@@ -102,24 +104,25 @@ class DoiEvent(str, Enum):
 class Doi:
     """The dataclass definition for a Doi object."""
 
+    # 🤔 TODO: replace the ``Any`` with what should be in aggregate types:
     title: str
     publication_date: datetime
     product_type: ProductType
     product_type_specific: str
     related_identifier: str
-    identifiers: list = field(default_factory=list)
-    authors: list = None
-    keywords: set = field(default_factory=set)
-    editors: list = None
-    description: str = None
-    id: str = None
-    doi: str = None
-    site_url: str = None
-    publisher: str = None
-    contributor: str = None
-    status: DoiStatus = None
-    previous_status: DoiStatus = None
-    message: str = None
-    date_record_added: datetime = None
-    date_record_updated: datetime = None
-    event: DoiEvent = None
+    identifiers: list[Any] = field(default_factory=list)
+    authors: Optional[list[Any]] = None
+    keywords: set[Any] = field(default_factory=set)
+    editors: Optional[list[Any]] = None
+    description: Optional[str] = None
+    id: Optional[str] = None
+    doi: Optional[str] = None
+    site_url: Optional[str] = None
+    publisher: Optional[str] = None
+    contributor: Optional[str] = None
+    status: Optional[DoiStatus] = None
+    previous_status: Optional[DoiStatus] = None
+    message: Optional[str] = None
+    date_record_added: Optional[datetime] = None
+    date_record_updated: Optional[datetime] = None
+    event: Optional[DoiEvent] = None

src/pds_doi_service/core/input/input_util.py

@@ -32,7 +32,7 @@
 from pds_doi_service.core.outputs.service import SERVICE_TYPE_DATACITE
 from pds_doi_service.core.util.config_parser import DOIConfigUtil
 from pds_doi_service.core.util.general_util import get_logger
-from xmlschema import XMLSchemaValidationError
+from xmlschema import XMLSchemaValidationError  # type: ignore
 
 # Get the common logger
 logger = get_logger(__name__)

src/pds_doi_service/core/input/test/read_remote_bundle.py

@@ -1,5 +1,8 @@
 import logging
 import time
+from urllib.request import urlopen
+
+import requests
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -8,7 +11,6 @@
 target_url = "https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_JSON_1D00.JSON"
 
 # fast method
-from urllib.request import urlopen
 
 timer_start = time.time()
 logger.info(f"TIMER_START:urlopen {target_url}")
@@ -22,7 +24,6 @@
 
 # slow method
 logger.info("==============================")
-import requests
 
 timer_start = time.time()
 session = requests.session()

src/pds_doi_service/core/outputs/datacite/__init__.py

@@ -6,7 +6,7 @@
 This package contains the DataCite-specific implementations for the abstract
 classes of the outputs package.
 """
-from .datacite_record import DOIDataCiteRecord
-from .datacite_validator import DOIDataCiteValidator
-from .datacite_web_client import DOIDataCiteWebClient
-from .datacite_web_parser import DOIDataCiteWebParser
+from .datacite_record import DOIDataCiteRecord  # noqa: F401
+from .datacite_validator import DOIDataCiteValidator  # noqa: F401
+from .datacite_web_client import DOIDataCiteWebClient  # noqa: F401
+from .datacite_web_parser import DOIDataCiteWebParser  # noqa: F401

src/pds_doi_service/core/outputs/datacite/datacite_web_parser.py

@@ -165,7 +165,7 @@ def _parse_date_record_added(record):
     def _parse_date_record_updated(record):
         try:
             return isoparse(record["updated"])
-        except (KeyError, ValueError) as err:
+        except (KeyError, ValueError):
             logger.warning('Could not parse optional field "date_record_updated"')
 
     @staticmethod

src/pds_doi_service/core/outputs/doi_record.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2020-21, by the California Institute of Technology.  ALL RIGHTS
+#  Copyright 2021–21, by the California Institute of Technology.  ALL RIGHTS
 #  RESERVED. United States Government Sponsorship acknowledged. Any commercial
 #  use must be negotiated with the Office of Technology Transfer at the
 #  California Institute of Technology.