Skip to content

Commit

Permalink
ci: performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@afuetterer
afuetterer committed Jul 16, 2024
1 parent a6cb40f commit 35cc578
Showing 1 changed file with 108 additions and 86 deletions.
194 changes: 108 additions & 86 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,35 +48,112 @@ jobs:
with:
extra_args: --all-files --color=always

build-wheel:
name: Build python wheel
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
# update the version
- name: Get short commit SHA
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
SHA="${{ github.event.pull_request.head.sha }}"
else
SHA="${{ github.sha }}"
fi
echo "SHA=$(git rev-parse --short $SHA)" >> $GITHUB_ENV
- name: Get current version (MAJOR.MINOR.PATCH)
id: current-version
run: echo "current_version=$(grep -Po '(?<=__version__ = ")[\d\w.]+(?=")' rdmo/__init__.py)" >> $GITHUB_OUTPUT
- name: Generate new version (current version + SHA)
id: new-version
run: echo "new_version=${{ steps.current-version.outputs.current_version }}+$SHA" >> $GITHUB_OUTPUT
- name: Update version in rdmo/__init__.py
run: |
sed -i "s/__version__ = .*/__version__ = \"${{ steps.new-version.outputs.new_version }}\"/" rdmo/__init__.py
# build the webpack bundle
- uses: actions/setup-node@v4
with:
node-version: 18
cache: npm
- run: npm install && npm run build:prod
# build the wheel
- uses: actions/setup-python@v5
with:
python-version: '3.12'
cache: pip
- run: |
python -m pip install --upgrade pip pipx
pip --version
pipx --version
- name: Build the wheel
run: pipx run build
- name: Check the metadata
run: pipx run twine check --strict dist/*
- name: Install package from built wheel
run: python -m pip install dist/rdmo*.whl
- name: Write info to step summary
run: |
{
echo -e "# ✓ Wheel successfully built (v${{ steps.new-version.outputs.new_version }})\n\n"
echo '<details><summary>Information about installed wheel</summary>'
echo -e "\n\`\`\`console"
echo "$ python -m pip show --files --verbose rdmo"
python -m pip show --files --verbose rdmo
echo -e "\`\`\`\n</details>"
} >> $GITHUB_STEP_SUMMARY
- name: Upload wheel as artifact
uses: actions/upload-artifact@v4
with:
name: wheel
path: dist/rdmo*.whl
if-no-files-found: error
retention-days: 30

test:
name: "Test (Python: ${{ matrix.python-version }}, DB: ${{ matrix.db-backend }})"
needs: build-wheel
runs-on: ubuntu-24.04
timeout-minutes: 45
strategy:
matrix:
python-version: ['3.8', '3.12']
db-backend: [mysql, postgres]
name: "Test (Python: ${{ matrix.python-version }}, DB: ${{ matrix.db-backend }})"
needs: lint
end-to-end: [false]
include:
- python-version: '3.12'
db-backend: postgres
end-to-end: true
steps:
- run: echo "true"
if: matrix.end-to-end
- run: echo "false"
if: matrix.end-to-end == false
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Download wheel
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Install Dependencies
run: |
sudo apt update
sudo apt install --yes pandoc texlive-xetex librsvg2-bin
python -m pip install --upgrade pip
sudo apt-get update && sudo apt-get install --yes pandoc texlive-xetex librsvg2-bin
pandoc --version
- name: Install rdmo[mysql] and start mysql
python -m pip install --upgrade pip
python -m pip --version
- name: Install rdmo[mysql] from wheel and start mysql
run: |
python -m pip install --editable .[ci,mysql]
python -m pip install "$(ls dist/*.whl)[ci,mysql]"
sudo systemctl start mysql.service
if: matrix.db-backend == 'mysql'
- name: Install rdmo[postgres] and start postgresql
- name: Install rdmo[postgres] from wheel and start postgresql
run: |
python -m pip install --editable .[ci,postgres]
python -m pip install "$(ls dist/*.whl)[ci,postgres]"
sudo systemctl start postgresql.service
pg_isready
sudo -u postgres psql --command="CREATE USER postgres_user PASSWORD 'postgres_password' CREATEDB"
Expand All @@ -89,41 +166,36 @@ jobs:
run: |
pytest rdmo/core/tests/test_package_status.py::test_package_json_and_pre_commit_versions_match \
--nomigrations --verbose
if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres'
if: matrix.end-to-end
- name: Run Tests
run: |
pytest -p randomly -p no:cacheprovider --cov --reuse-db --numprocesses=auto --dist=loadscope
env:
GITHUB_DB_BACKEND: ${{ matrix.db-backend }}
if: matrix.end-to-end == false
- name: Upload coverage data to coveralls.io
uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
with:
flag-name: '${{ matrix.db-backend }}: ${{ matrix.python-version }}'
parallel: true
if: matrix.end-to-end == false
# end-to-end tests
- uses: actions/setup-node@v4
with:
node-version: 18
cache: npm
if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres'
- name: Install e2e tests dependencies
run: |
npm install
npm run build:prod
playwright install chromium
if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres'
- run: mkdir screenshots
run: playwright install chromium
if: matrix.end-to-end
- name: Run end-to-end tests
run: pytest -p randomly -p no:cacheprovider --reuse-db --numprocesses=auto --dist=loadscope -m e2e --nomigrations
if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres'
run: |
mkdir screenshots
pytest -p randomly -p no:cacheprovider --reuse-db --numprocesses=auto --dist=loadscope -m e2e --nomigrations
if: matrix.end-to-end
env:
DJANGO_DEBUG: True
GITHUB_DB_BACKEND: ${{ matrix.db-backend }}
- uses: actions/upload-artifact@v4
with:
name: screenshots
path: screenshots/*.png
if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres'
if: matrix.end-to-end

coveralls:
name: Indicate completion to coveralls
Expand All @@ -136,63 +208,6 @@ jobs:
with:
parallel-finished: true

build-wheel:
name: Build python wheel
needs: test
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- name: Get short commit SHA
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
SHA="${{ github.event.pull_request.head.sha }}"
else
SHA="${{ github.sha }}"
fi
echo "SHA=$(git rev-parse --short $SHA)" >> $GITHUB_ENV
- name: Get current version (MAJOR.MINOR.PATCH)
id: current-version
run: echo "current_version=$(grep -Po '(?<=__version__ = ")[\d\w.]+(?=")' rdmo/__init__.py)" >> $GITHUB_OUTPUT
- name: Generate new version (current version + SHA)
id: new-version
run: echo "new_version=${{ steps.current-version.outputs.current_version }}+$SHA" >> $GITHUB_OUTPUT
- name: Update version in rdmo/__init__.py
run: |
sed -i "s/__version__ = .*/__version__ = \"${{ steps.new-version.outputs.new_version }}\"/" rdmo/__init__.py
- uses: actions/setup-node@v4
with:
node-version: 18
cache: npm
- run: npm install
- run: npm run build:prod
- uses: actions/setup-python@v5
with:
python-version: '3.12'
cache: pip
- run: |
python -m pip install --upgrade pip
python -m pip install .[dev]
- name: Build the wheel
run: python -m build --wheel
- name: Check metadata
run: python -m twine check --strict dist/*
- name: Install package from built wheel
run: python -m pip install --force-reinstall dist/rdmo*.whl
- name: Write info to step summary
run: |
echo -e "# ✓ Wheel successfully built (v${{ steps.new-version.outputs.new_version }})\n\n" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`console" >> $GITHUB_STEP_SUMMARY
echo "$ python -m pip show rdmo" >> $GITHUB_STEP_SUMMARY
python -m pip show rdmo >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
- name: Upload wheel as artifact
uses: actions/upload-artifact@v4
with:
name: wheel
path: dist/rdmo*.whl
if-no-files-found: error
retention-days: 30

dev-setup:
# Ref: structlog (MIT licensed) <https://github.com/hynek/structlog/blob/main/.github/workflows/ci.yml>
name: "Test dev setup on ${{ matrix.os }}"
Expand All @@ -211,19 +226,26 @@ jobs:

dependencies:
name: Test installation of all dependencies
needs: build-wheel
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
- name: Download wheel
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Install os requirements for python-ldap
run: |
sudo apt update
sudo apt install --yes libldap2-dev libsasl2-dev
run: sudo apt-get update && sudo apt-get install --yes libldap2-dev libsasl2-dev
- run: python -m pip install --upgrade pip
- run: python -m pip install .[allauth,ci,dev,gunicorn,ldap,mysql,postgres,pytest]
- name: Install rdmo wheel with all optional dependency groups
run: python -m pip install "$(ls dist/*.whl)[allauth,ci,dev,gunicorn,ldap,mysql,postgres,pytest]"
- name: Verify installed packages have compatible dependencies
run: python -m pip check
- uses: actions/setup-node@v4
with:
node-version: 18
Expand Down Expand Up @@ -259,9 +281,9 @@ jobs:
if: always()
needs:
- lint
- build-wheel
- test
- coveralls
- build-wheel
- dev-setup
- dependencies
runs-on: ubuntu-24.04
Expand Down

0 comments on commit 35cc578

Please sign in to comment.