add pandas 2.2.0 to tests, use uv for pip compile (#1502)

* add pandas 2.2.0 to tests, use uv for pip compile Signed-off-by: cosmicBboy <[email protected]> * fix ci Signed-off-by: cosmicBboy <[email protected]> * update nox tests and ci Signed-off-by: cosmicBboy <[email protected]> * reformat Signed-off-by: cosmicBboy <[email protected]> * fix ci formatting Signed-off-by: cosmicBboy <[email protected]> * fix typo Signed-off-by: cosmicBboy <[email protected]> * fix extras typo Signed-off-by: cosmicBboy <[email protected]> * only install uv in venv Signed-off-by: cosmicBboy <[email protected]> * limit dask tests to pandas 1.5.3 Signed-off-by: cosmicBboy <[email protected]> --------- Signed-off-by: cosmicBboy <[email protected]>
unionai-oss · Feb 22, 2024 · e697eb2 · e697eb2
1 parent c79ccc7
commit e697eb2
Show file tree

Hide file tree

Showing 35 changed files with 15,481 additions and 4,450 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -70,35 +70,39 @@ make docs
 
 #### Adding New Dependencies
 
-To add new dependencies to the project, make sure to alter the _environment.yml_ file. Then to sync the dependencies from the _environment.yml_ file to the _requirements-dev.txt_ run the following command
+This repo uses [mamba](https://github.com/mamba-org/mamba), which is a faster
+implementation of [miniconda](https://docs.conda.io/en/latest/miniconda.html),
+to run the `nox` test suite. Simply install it via conda-forge:
 
 ```bash
-make requirements-dev.txt
+conda install -c conda-forge mamba
+```
+
+To add new dependencies to the project, first alter the _environment.yml_ file. Then to sync the dependencies from the `environment.yml`` file to the `requirements.in` run the following command
+
+```bash
+make nox-ci-requirements nox-dev-requirements
 ```
 
 This will:
 
 - Invoke `python scripts/generate_pip_deps_from_conda.py` to convert `environment.yml`
   to a `requirements.in` file.
-- Use `pip-compile` to create `requirements-dev.txt` file that has a fully specified
-  set of dependencies.
+- Use `pip-compile` via the `uv` package to create requirements files in the
+  `ci` and `dev` directories. The `ci` requirements files are used by github
+   actions, while those in the `dev` directory should be used to create local
+   development enviornments.
 
-You can use the resulting `requirements-dev.txt` file to install your dependencies
+You can use the resulting `requirements-{3.x}.txt` file to install your dependencies
 with `pip`:
 
 ```bash
-pip install -r requirements-dev.txt
+pip install -r dev/requirements-{3.x}.txt  # replace {3.x} with desired python version
 ```
 
 Moreover to add new extra dependencies in setup.py, it is necessary to add it to
 the **_extras_require** dictionary.
 
-When you update dependencies also need to update the `pip-compile`d requirements
-files in the `ci` directory, which are used by the CI/CD process of this repo:
-
-```bash
-make nox-ci-requirements
-```
 
 #### Set up `pre-commit`
 
@@ -150,21 +154,6 @@ make nox-conda
 make nox
 ```
 
-Option 2 assumes that you have python environments for all of the versions
-that pandera supports.
-
-#### Using `mamba` (optional)
-
-You can also use [mamba](https://github.com/mamba-org/mamba), which is a faster
-implementation of [miniconda](https://docs.conda.io/en/latest/miniconda.html),
-to run the `nox` test suite. Simply install it via conda-forge, and
-`make nox-conda` should use it under the hood.
-
-```bash
-conda install -c conda-forge mamba
-make nox-conda
-```
-
 ### Project Releases
 
 Releases are organized under [milestones](https://github.com/pandera-dev/pandera/milestones),

diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -101,7 +101,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        pandas-version: ["1.5.3", "2.0.3"]
+        pandas-version: ["1.5.3", "2.0.3", "2.2.0"]
         pydantic-version: ["1.10.11", "2.3.0"]
         include:
         - os: ubuntu-latest
@@ -110,6 +110,9 @@ jobs:
           pip-cache: ~/Library/Caches/pip
         - os: windows-latest
           pip-cache: ~/AppData/Local/pip/Cache
+        exclude:
+        - python-version: "3.8"
+          pandas-version: "2.2.0"
 
     steps:
       - uses: actions/checkout@v4
@@ -119,16 +122,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Cache conda
-        uses: actions/cache@v4
-        env:
-          # Increase this value to reset cache if etc/environment.yml has not changed
-          CACHE_NUMBER: 2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
-        id: cache
-
       - name: Cache pip
         uses: actions/cache@v4
         with:
@@ -138,59 +131,57 @@ jobs:
             ${{ runner.os }}-pip-
 
       - name: Install deps
-        run: pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt
+        run:
+          pip install uv
+          uv pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt
 
       - run: |
           pip list
           printenv | sort
 
       - name: Unit Tests - Core
-        run: pytest tests/core ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='core', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Hypotheses
-        run: pytest tests/hypotheses ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='hypotheses', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - IO
-        run: pytest tests/io ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='io', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Mypy
         # mypy tests hang on windows
         if: ${{ matrix.os != 'windows-latest' }}
-        run: pytest -v tests/mypy ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='mypy', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Strategies
-        run: pytest tests/strategies ${{ env.PYTEST_FLAGS }} ${{ env.HYPOTHESIS_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='strategies', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - FastAPI
         # there's an issue with the fastapi tests in CI that's not reproducible locally
         # when pydantic > v2
-        run: pytest tests/fastapi ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='fastapi', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - GeoPandas
-        run: pytest tests/geopandas ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='geopandas', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Dask
-        if: ${{ matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/dask ${{ env.PYTEST_FLAGS }}
+        if: ${{ matrix.pandas-version == '1.5.3' }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='dask', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Pyspark
-        if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/pyspark ${{ env.PYTEST_FLAGS }}
+        if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version == '1.5.3' }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='pyspark', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Modin-Dask
-        if: ${{ matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
-        env:
-          CI_MODIN_ENGINES: dask
+        if: ${{ matrix.pandas-version == '1.5.3' }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='modin-dask', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Modin-Ray
         # ray CI issues with the following:
         # - windows, python 3.10
         # Tracking issue: https://github.com/modin-project/modin/issues/5466
-        if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
-        env:
-          CI_MODIN_ENGINES: ray
+        if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version == '1.5.3' }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='modin-ray', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,13 +44,17 @@ repos:
         args: ["--disable=import-error"]
         exclude: (^docs/|^scripts)
 
-  - repo: local
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.982
     hooks:
       - id: mypy
-        name: mypy
-        entry: mypy
-        language: python
-        types: [python]
+        additional_dependencies:
+          - numpy
+          - types-click
+          - types-pkg_resources
+          - types-pytz
+          - types-pyyaml
+          - types-requests
         args: ["pandera", "tests", "scripts"]
         exclude: (^docs/|^tests/mypy/modules/)
         pass_filenames: false

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -24,6 +24,6 @@ formats: []
 # Optionally set the version of Python and requirements required to build your docs
 python:
   install:
-    - requirements: requirements-docs.txt
+    - requirements: dev/requirements-3.10.txt
     - method: pip
       path: .
diff --git a/Makefile b/Makefile
@@ -38,17 +38,17 @@ nox:
 
 NOX_FLAGS ?= "-r"
 
-nox-conda:
-	nox -db conda --envdir .nox-conda ${NOX_FLAGS}
+nox-mamba:
+	nox -db mamba --envdir .nox-mamba ${NOX_FLAGS}
 
 deps-from-conda:
 	python scripts/generate_pip_deps_from_conda.py
 
 nox-ci-requirements: deps-from-conda
-	nox -db mamba --envdir .nox-mamba -s ci_requirements
+	nox -db mamba --envdir .nox-mamba -s ci_requirements ${NOX_FLAGS}
 
 nox-dev-requirements: deps-from-conda
-	nox -db mamba --envdir .nox-mamba -s dev_requirements
+	nox -db mamba --envdir .nox-mamba -s dev_requirements ${NOX_FLAGS}
 
-requirements-docs.txt: deps-from-conda
-	pip-compile requirements.in --no-emit-index-url --output-file requirements-docs.txt -v --resolver backtracking
+nox-tests:
+	nox -db mamba --envdir .nox-mamba -s tests ${NOX_FLAGS}