Integration tests (#228)

* misc: extreme amounts of linting

* fix: fix failing tests

* chore: bump version

* feat: add __all__ automatically for proper wildcard imports

* fix: update pre-commit github action version

* fix: update python version in all files

* chore: refactor to enable integration testing

* chore: further refactoring

* fix: remove _all_ (broke rename symbol in vscode)

* chore: refactor for integration testing

* restructure: move synth data generation into dir with files

* feat: update synth datasets

* feat: ignore unused imports

* refactor and pass tests!

* feat: add integration tests from main script

* feat: remove version number when running pytests for caching

* fix: correct ordering in pytest

* debug: dir structure in github actions

* same

* fix: update sed command

* update sed command

* fix: update sed command

* standard sed

* feat: echo for debugging

* fix: correct id for action

* fix: incorrect id in github action

* fix: actually print toml contents

* test new cach

* feat: run pytest in parallel

* feat: parallellise by using pytest tmp_path

* test: lasses version bump induced failing tests, see if propagating

* feat: add dask for faster concatenation

* fix: clean up merge errors

* chore: bunch of type checking

* misc: fixes for overtaci

* feat: add cache test for binary features

* misc: remove outlier sample detection

* fix: tiny fixes

* fix: fix failing tests

* fix: review comments*

* feat: re-add Path

* feat: add PROJECT_ROOT

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = C901, E203, E501, W503, W291
+ignore = C901, E203, E501, F401, W503, W291, 
 select = B,C,E,F,W,T4
 exclude =
     .env,

.github/workflows/pytest.yml

@@ -23,6 +23,12 @@ jobs:
       uses: actions/checkout@v3
 
     ### Caching
+    - name: Overwrite version number from pyproject.toml, so it doesn't invalidate cache
+      id: remove-version-from-toml
+      run: |
+        sed -i "s/version = \".*\"/version = \"0.0.0\"/g" pyproject.toml
+        cat pyproject.toml
+        
     - name: Load cached $HOME/.local (cache poetry)
       uses: actions/[email protected]
       with:
@@ -51,6 +57,11 @@ jobs:
       run: poetry install
 
     ### Run tests
+    - name: Parralelise pytest
+      run: |
+        echo -e "[pytest]\naddopts='-n 2'" > pytest.ini
+        cat pytest.ini
+
     - name: Source .venv
       run: |
         source $VENV

.pre-commit-config.yaml

@@ -17,6 +17,7 @@ repos:
     rev: 1208357bcc569b86a4a0dc69b108cbe28f190f54
     hooks:
     - id: pybetter
+      args: [--select, "B002,B007"]
 
   - repo: https://github.com/bwhmather/ssort
     rev: v0.11.6

docs/conf.py

@@ -96,20 +96,3 @@
     "navigation_with_keys": True,
     "top_of_page_button": "edit",
 }
-
-
-__all__ = [
-    "author",
-    "exclude_patterns",
-    "extensions",
-    "html_context",
-    "html_favicon",
-    "html_show_sourcelink",
-    "html_static_path",
-    "html_theme",
-    "html_theme_options",
-    "project",
-    "release",
-    "source_suffix",
-    "templates_path",
-]

example/loaders/debug_loader_function.py

@@ -94,11 +94,3 @@ def get_prop_of_each_unique_value_for_non_floats(series: pd.Series) -> pd.Series
         subset_duplicates_columns=["dw_ek_borger", "timestamp", "value"],
     )
     print(errors)
-
-
-__all__ = [
-    "df",
-    "get_prop_of_each_unique_value_for_non_floats",
-    "value_props",
-    "will_it_float",
-]

example/loaders/load_coercion.py

@@ -8,10 +8,3 @@
     df = c.coercion_duration(n_rows=100)
     farlighed = c.farlighed(n_rows=20)
     baelte = c.baelte(n_rows=100)
-
-
-__all__ = [
-    "baelte",
-    "df",
-    "farlighed",
-]

example/loaders/load_diagnoses.py

@@ -14,9 +14,3 @@
         predictor_dict_list=input_dict,
         n_rows=100,
     )
-
-
-__all__ = [
-    "df",
-    "input_dict",
-]

example/loaders/load_medications.py

@@ -4,8 +4,3 @@
 
 if __name__ == "__main__":
     df = m.antipsychotics()
-
-
-__all__ = [
-    "df",
-]

example/loaders/load_physical_visits.py

@@ -3,8 +3,3 @@
 
 if __name__ == "__main__":
     df = r.load_visits.physical_visits_to_psychiatry()
-
-
-__all__ = [
-    "df",
-]

example/loaders/load_sql.py

@@ -6,10 +6,3 @@
     VIEW = "[psycop_t2d_train]"
     SQL = "SELECT * FROM [fct]." + VIEW
     df = sql_load(SQL, chunksize=None, format_timestamp_cols_to_datetime=False)
-
-
-__all__ = [
-    "SQL",
-    "VIEW",
-    "df",
-]

pyproject.toml

@@ -33,6 +33,7 @@ deepchecks = "^0.8.3"
 psutil = "5.9.1"
 dill = "^0.3.5"
 
+
 [tool.poetry.dev-dependencies]
 black = "22.3.*"
 pre-commit = "2.19.*"
@@ -48,7 +49,11 @@ docformatter = "1.4"
 darglint = "^1.8.1"
 SQLAlchemy = "^1.4.40"
 pyodbc = "^4.0.34"
+tqdm = "^4.64.1"
+dask = "^2022.9.0"
 transformers = "^4.22.0"
+mypy = "^0.971"
+pytest-xdist = "^2.5.0"
 
 [tool.coverage.run]
 omit = [
@@ -90,4 +95,6 @@ known_third_party = ["wandb"]
 [tool.pylint]
 load-plugins = "pylint.extensions.docparams,pylint.extensions.code_style,pylint.extensions.for_any_all,pylint.extensions.typing"
 good-names = "df,p,f,d,e,n,k,i,v"
-disable = "too-many-lines,line-too-long,missing-raises-doc,no-self-argument,unused-wildcard-import,wildcard-import,no-else-return,too-many-arguments"
+disable = "too-many-lines,line-too-long,missing-raises-doc,no-self-argument,unused-wildcard-import,wildcard-import,no-else-return,too-many-arguments,redefined-outer-name"
+
+[tool.pyright] # Added because of weird pylance error, issue here: https://github.com/microsoft/pylance-release/issues/3366

src/application/t2d/check_feature_set_integrity.py

@@ -3,20 +3,15 @@
 from pathlib import Path
 
 from psycopmlutils.data_checks.flattened.data_integrity import (
-    check_feature_set_integrity_from_dir,
+    save_feature_set_integrity_from_dir,
 )
 
 if __name__ == "__main__":
     subdir = Path(
         "E:/shared_resources/feature_sets/t2d/adminmanber_260_features_2022_08_26_14_10/",
     )
 
-    check_feature_set_integrity_from_dir(
+    save_feature_set_integrity_from_dir(
         feature_set_csv_dir=subdir,
         split_names=["train", "val", "test"],
     )
-
-
-__all__ = [
-    "subdir",
-]

src/application/t2d/describe_features.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 
 from psycopmlutils.data_checks.flattened.feature_describer import (
-    create_feature_description_from_dir,
+    save_feature_description_from_dir,
 )
 from src.application.t2d.features_blood_samples import get_lab_feature_spec
 from src.application.t2d.features_diagnoses import get_diagnosis_feature_spec
@@ -42,20 +42,7 @@
 
     PREDICTOR_LIST = MEDICATION_PREDICTORS + DIAGNOSIS_PREDICTORS + LAB_PREDICTORS
 
-    create_feature_description_from_dir(
-        path=feature_set_dir,
+    save_feature_description_from_dir(
+        feature_set_csv_dir=feature_set_dir,
         predictor_dicts=PREDICTOR_LIST,
     )
-
-
-__all__ = [
-    "DIAGNOSIS_PREDICTORS",
-    "LAB_PREDICTORS",
-    "MEDICATION_PREDICTORS",
-    "PREDICTOR_LIST",
-    "feature_set_dir",
-    "feature_set_path",
-    "lookbehind_days",
-    "out_dir",
-    "resolve_multiple",
-]

src/application/t2d/features_blood_samples.py

@@ -1,19 +1,21 @@
 """Feature specificatin for T2D blood samples."""
 
+from typing import Optional
+
 import numpy as np
 
 
 def get_lab_feature_spec(  # pylint: disable=dangerous-default-value
     # Not a problem since the function is only called once.
-    lookbehind_days: int = None,
-    resolve_multiple: str = None,
+    lookbehind_days: Optional[list[int]] = None,
+    resolve_multiple: Optional[list[str]] = None,
     values_to_load="all",
 ) -> list:
     """Get feature specification for T2D blood samples.
 
     Args:
-        lookbehind_days (int, optional): Defaults to None.
-        resolve_multiple (str, optional): Defaults to None.
+        lookbehind_days (list[int], optional): Defaults to None.
+        resolve_multiple (list[str], optional): Defaults to None.
         values_to_load (str, optional): Defaults to "all".
 
     Returns:
@@ -57,8 +59,3 @@ def get_lab_feature_spec(  # pylint: disable=dangerous-default-value
         }
         for df in dfs
     ]
-
-
-__all__ = [
-    "get_lab_feature_spec",
-]

src/application/t2d/features_diagnoses.py

@@ -48,8 +48,3 @@ def get_diagnosis_feature_spec(  # pylint: disable=dangerous-default-value
         }
         for df in dfs
     ]
-
-
-__all__ = [
-    "get_diagnosis_feature_spec",
-]

src/application/t2d/features_medications.py

@@ -1,17 +1,20 @@
 """Loaders for T2D medication feature spec."""
 
 
+from typing import Any, Optional
+
+
 def get_medication_feature_spec(  # pylint: disable=dangerous-default-value
-    lookbehind_days: int = None,
-    resolve_multiple: str = None,
-    fallback=0,
-):
+    lookbehind_days: Optional[list[int]] = None,
+    resolve_multiple: Optional[list[str]] = None,
+    fallback: Any = 0,
+) -> list:
     """Get feature specification for T2D medications.
 
     Args:
-        lookbehind_days (int, optional): Defaults to None.
-        resolve_multiple (str, optional): Defaults to None.
-        fallback (int, optional): Defaults to 0.
+        lookbehind_days (list[int], optional): Defaults to None.
+        resolve_multiple (list[str], optional): Defaults to None.
+        fallback (Any, optional): Defaults to 0.
 
     Returns:
         list: Feature specification.
@@ -32,8 +35,3 @@ def get_medication_feature_spec(  # pylint: disable=dangerous-default-value
             "allowed_nan_value_prop": 0.0,
         },
     ]
-
-
-__all__ = [
-    "get_medication_feature_spec",
-]