diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 921c235d4c..4d82dc43d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,18 +34,18 @@ env: jobs: - lint: - # Ref: https://github.com/rdmorganiser/.github/blob/main/.github/workflows/_lint.yml - uses: rdmorganiser/.github/.github/workflows/_lint.yml@main + # lint: + # # Ref: https://github.com/rdmorganiser/.github/blob/main/.github/workflows/_lint.yml + # uses: rdmorganiser/.github/.github/workflows/_lint.yml@main test: runs-on: ubuntu-24.04 strategy: matrix: - python-version: ['3.8', '3.12'] - db-backend: [mysql, postgres] + python-version: ['3.12'] + db-backend: [postgres] name: "Test (Python: ${{ matrix.python-version }}, DB: ${{ matrix.db-backend }})" - needs: lint + # needs: lint steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -67,6 +67,7 @@ jobs: - name: Install rdmo[postgres] and start postgresql run: | python -m pip install --editable .[ci,postgres] + python -m pip install pytest-repeat sudo systemctl start postgresql.service pg_isready sudo -u postgres psql --command="CREATE USER postgres_user PASSWORD 'postgres_password' CREATEDB" @@ -75,192 +76,193 @@ jobs: run: | cp -r testing/media testing/media_root mkdir testing/log - - name: Run package status tests first + - name: Run tests that use the "files" fixture run: | - pytest rdmo/core/tests/test_package_status.py::test_package_json_and_pre_commit_versions_match \ - --nomigrations --verbose - if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' - - name: Run Tests + pytest -p randomly --nomigrations --verbose --numprocesses=auto --dist=loadscope --count=10 -x -m files + - name: Run a test function that definitively failed once run: | - pytest -p randomly -p no:cacheprovider --cov --reuse-db --numprocesses=auto --dist=loadscope - env: - GITHUB_DB_BACKEND: ${{ matrix.db-backend }} - - name: Upload coverage data to coveralls.io - uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0 - with: - flag-name: '${{ matrix.db-backend }}: ${{ matrix.python-version }}' - parallel: true - # end-to-end tests - - uses: actions/setup-node@v4 - with: - node-version: 18 - cache: npm - if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' - - name: Install e2e tests dependencies - run: | - npm install - npm run build:prod - playwright install chromium - if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' - - run: mkdir screenshots - - name: Run end-to-end tests - run: pytest -p randomly -p no:cacheprovider --reuse-db --numprocesses=auto --dist=loadscope -m e2e --nomigrations - if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' - env: - DJANGO_DEBUG: True - GITHUB_DB_BACKEND: ${{ matrix.db-backend }} - - uses: actions/upload-artifact@v4 - with: - name: screenshots - path: screenshots/*.png - if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' + pytest -p randomly --nomigrations --verbose --numprocesses=auto --dist=loadscope --count=100 -x rdmo/projects/tests/test_view_project_create_import.py::test_project_create_import_post_import_file + # - name: Run Tests + # run: | + # pytest -p randomly -p no:cacheprovider --cov --reuse-db --numprocesses=auto --dist=loadscope + # env: + # GITHUB_DB_BACKEND: ${{ matrix.db-backend }} + # - name: Upload coverage data to coveralls.io + # uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0 + # with: + # flag-name: '${{ matrix.db-backend }}: ${{ matrix.python-version }}' + # parallel: true + # # end-to-end tests + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + # cache: npm + # if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' + # - name: Install e2e tests dependencies + # run: | + # npm install + # npm run build:prod + # playwright install chromium + # if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' + # - run: mkdir screenshots + # - name: Run end-to-end tests + # run: pytest -p randomly -p no:cacheprovider --reuse-db --numprocesses=auto --dist=loadscope -m e2e --nomigrations + # if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' + # env: + # DJANGO_DEBUG: True + # GITHUB_DB_BACKEND: ${{ matrix.db-backend }} + # - uses: actions/upload-artifact@v4 + # with: + # name: screenshots + # path: screenshots/*.png + # if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres' - coveralls: - name: Indicate completion to coveralls - needs: test - if: ${{ always() }} - runs-on: ubuntu-24.04 - steps: - - name: Run Coveralls finish - uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0 - with: - parallel-finished: true + # coveralls: + # name: Indicate completion to coveralls + # needs: test + # if: ${{ always() }} + # runs-on: ubuntu-24.04 + # steps: + # - name: Run Coveralls finish + # uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0 + # with: + # parallel-finished: true - build-wheel: - name: Build python wheel - needs: test - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - # update the version - - name: Get short commit SHA - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - SHA="${{ github.event.pull_request.head.sha }}" - else - SHA="${{ github.sha }}" - fi - echo "SHA=$(git rev-parse --short $SHA)" >> $GITHUB_ENV - - name: Get current version (MAJOR.MINOR.PATCH) - id: current-version - run: echo "current_version=$(grep -Po '(?<=__version__ = ")[\d\w.]+(?=")' rdmo/__init__.py)" >> $GITHUB_OUTPUT - - name: Generate new version (current version + SHA) - id: new-version - run: echo "new_version=${{ steps.current-version.outputs.current_version }}+$SHA" >> $GITHUB_OUTPUT - - name: Update version in rdmo/__init__.py - run: | - sed -i "s/__version__ = .*/__version__ = \"${{ steps.new-version.outputs.new_version }}\"/" rdmo/__init__.py - # build the webpack bundle - - uses: actions/setup-node@v4 - with: - node-version: 18 - cache: npm - - run: npm install && npm run build:prod - # build the wheel - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: pip - - run: | - python -m pip install --upgrade pip build[uv] twine - pip --version - - name: Build the wheel - run: python -m build --installer=uv - - name: Check the metadata of wheel and sdist - run: python -m twine check --strict dist/* - - name: Install package from built wheel - run: python -m pip install --no-compile dist/rdmo*.whl # do not create __pycache__/*.pyc files - - name: Write info to step summary - run: | - { - echo -e "# ✓ Wheel successfully built (v${{ steps.new-version.outputs.new_version }})\n\n" - echo '
Information about installed wheel' - echo -e "\n\`\`\`console" - echo "$ python -m pip show --files --verbose rdmo" - python -m pip show --files --verbose rdmo - echo -e "\`\`\`\n
" - } >> $GITHUB_STEP_SUMMARY - - name: Upload wheel as artifact - uses: actions/upload-artifact@v4 - with: - name: wheel - path: dist/rdmo*.whl - if-no-files-found: error - retention-days: 30 + # build-wheel: + # name: Build python wheel + # needs: test + # runs-on: ubuntu-24.04 + # steps: + # - uses: actions/checkout@v4 + # # update the version + # - name: Get short commit SHA + # run: | + # if [ "${{ github.event_name }}" = "pull_request" ]; then + # SHA="${{ github.event.pull_request.head.sha }}" + # else + # SHA="${{ github.sha }}" + # fi + # echo "SHA=$(git rev-parse --short $SHA)" >> $GITHUB_ENV + # - name: Get current version (MAJOR.MINOR.PATCH) + # id: current-version + # run: echo "current_version=$(grep -Po '(?<=__version__ = ")[\d\w.]+(?=")' rdmo/__init__.py)" >> $GITHUB_OUTPUT + # - name: Generate new version (current version + SHA) + # id: new-version + # run: echo "new_version=${{ steps.current-version.outputs.current_version }}+$SHA" >> $GITHUB_OUTPUT + # - name: Update version in rdmo/__init__.py + # run: | + # sed -i "s/__version__ = .*/__version__ = \"${{ steps.new-version.outputs.new_version }}\"/" rdmo/__init__.py + # # build the webpack bundle + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + # cache: npm + # - run: npm install && npm run build:prod + # # build the wheel + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.12' + # cache: pip + # - run: | + # python -m pip install --upgrade pip build[uv] twine + # pip --version + # - name: Build the wheel + # run: python -m build --installer=uv + # - name: Check the metadata of wheel and sdist + # run: python -m twine check --strict dist/* + # - name: Install package from built wheel + # run: python -m pip install --no-compile dist/rdmo*.whl # do not create __pycache__/*.pyc files + # - name: Write info to step summary + # run: | + # { + # echo -e "# ✓ Wheel successfully built (v${{ steps.new-version.outputs.new_version }})\n\n" + # echo '
Information about installed wheel' + # echo -e "\n\`\`\`console" + # echo "$ python -m pip show --files --verbose rdmo" + # python -m pip show --files --verbose rdmo + # echo -e "\`\`\`\n
" + # } >> $GITHUB_STEP_SUMMARY + # - name: Upload wheel as artifact + # uses: actions/upload-artifact@v4 + # with: + # name: wheel + # path: dist/rdmo*.whl + # if-no-files-found: error + # retention-days: 30 - dev-setup: - # Ref: structlog (MIT licensed) - name: "Test dev setup on ${{ matrix.os }}" - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - - run: python -Im pip install --editable .[dev] - - run: python -Ic 'import rdmo; print(rdmo.__version__)' + # dev-setup: + # # Ref: structlog (MIT licensed) + # name: "Test dev setup on ${{ matrix.os }}" + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # os: [ubuntu-latest, windows-latest, macos-latest] + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-python@v5 + # with: + # python-version: "3.12" + # cache: pip + # - run: python -Im pip install --editable .[dev] + # - run: python -Ic 'import rdmo; print(rdmo.__version__)' - dependencies: - name: Test installation of all dependencies - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - - name: Install os requirements for python-ldap - run: | - sudo apt update - sudo apt install --yes libldap2-dev libsasl2-dev - - run: python -m pip install --upgrade pip - - run: python -m pip install .[allauth,ci,dev,gunicorn,ldap,mysql,postgres,pytest] - - uses: actions/setup-node@v4 - with: - node-version: 18 - cache: npm - - run: npm install --include=dev - - name: Write info to step summary - run: | - { - echo -e "# ✓ All dependency groups successfully installed in combination\n\n" - echo '
Installed Python packages (dependency tree)' - echo -e "\n\`\`\`console" - echo "$ python -m pipdeptree --local-only --exclude=pip,pipdeptree" - python -m pipdeptree --local-only --exclude=pip,pipdeptree - echo -e "\`\`\`\n
" - echo '
Outdated Python dependencies' - echo -e "\n\`\`\`console" - echo "$ python -m pip list --outdated" - python -m pip list --outdated - echo -e "\`\`\`\n
" - echo '
Installed JavaScript packages (dependency tree)' - echo -e "\n\`\`\`console" - echo "$ npm list --all" - npm list --all - echo -e "\`\`\`\n
" - echo '
Outdated JavaScript dependencies' - echo -e "\n\`\`\`console" - echo "$ npm outdated --long" - npm outdated --long || true - echo -e "\`\`\`\n
" - } >> $GITHUB_STEP_SUMMARY + # dependencies: + # name: Test installation of all dependencies + # runs-on: ubuntu-24.04 + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-python@v5 + # with: + # python-version: "3.12" + # cache: pip + # - name: Install os requirements for python-ldap + # run: | + # sudo apt update + # sudo apt install --yes libldap2-dev libsasl2-dev + # - run: python -m pip install --upgrade pip + # - run: python -m pip install .[allauth,ci,dev,gunicorn,ldap,mysql,postgres,pytest] + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + # cache: npm + # - run: npm install --include=dev + # - name: Write info to step summary + # run: | + # { + # echo -e "# ✓ All dependency groups successfully installed in combination\n\n" + # echo '
Installed Python packages (dependency tree)' + # echo -e "\n\`\`\`console" + # echo "$ python -m pipdeptree --local-only --exclude=pip,pipdeptree" + # python -m pipdeptree --local-only --exclude=pip,pipdeptree + # echo -e "\`\`\`\n
" + # echo '
Outdated Python dependencies' + # echo -e "\n\`\`\`console" + # echo "$ python -m pip list --outdated" + # python -m pip list --outdated + # echo -e "\`\`\`\n
" + # echo '
Installed JavaScript packages (dependency tree)' + # echo -e "\n\`\`\`console" + # echo "$ npm list --all" + # npm list --all + # echo -e "\`\`\`\n
" + # echo '
Outdated JavaScript dependencies' + # echo -e "\n\`\`\`console" + # echo "$ npm outdated --long" + # npm outdated --long || true + # echo -e "\`\`\`\n
" + # } >> $GITHUB_STEP_SUMMARY - required-checks-pass: - if: always() - needs: - - lint - - test - - coveralls - - build-wheel - - dev-setup - - dependencies - runs-on: ubuntu-24.04 - steps: - - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} + # required-checks-pass: + # if: always() + # needs: + # - lint + # - test + # - coveralls + # - build-wheel + # - dev-setup + # - dependencies + # runs-on: ubuntu-24.04 + # steps: + # - uses: re-actors/alls-green@release/v1 + # with: + # jobs: ${{ toJSON(needs) }} diff --git a/conftest.py b/conftest.py index 5dbd3652e6..483d48b1b8 100644 --- a/conftest.py +++ b/conftest.py @@ -44,8 +44,8 @@ def django_db_setup(django_db_setup, django_db_blocker, fixtures): # noqa: PT004 @pytest.fixture def files(settings, tmp_path): """Create a temporary MEDIA_ROOT directory and copy test data into it.""" - media_path = Path(__file__).parent.joinpath("testing").joinpath("media") - settings.MEDIA_ROOT = tmp_path.joinpath("media") + media_path = Path(__file__).parent / "testing" / "media" + settings.MEDIA_ROOT = tmp_path / "media" shutil.copytree(media_path, settings.MEDIA_ROOT) return settings.MEDIA_ROOT diff --git a/pyproject.toml b/pyproject.toml index f8d45ed984..f7ca7525b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,7 @@ pythonpath = [".", "testing"] addopts = '-p no:randomly -m "not e2e"' markers = [ "e2e: marks tests as end-to-end tests using playwright (deselect with '-m \"not e2e\"')", + "files: marks tests that use the 'files' fixture", ] filterwarnings = [ # fail on RemovedInDjango50Warning exception diff --git a/rdmo/projects/tests/test_view_issue.py b/rdmo/projects/tests/test_view_issue.py index 8780c7114f..55f2091540 100644 --- a/rdmo/projects/tests/test_view_issue.py +++ b/rdmo/projects/tests/test_view_issue.py @@ -176,6 +176,7 @@ def test_issue_send_post_email(db, client, username, password, project_id, issue assert response.status_code == 404 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('issue_id', issues) @pytest.mark.parametrize('project_id', projects) diff --git a/rdmo/projects/tests/test_view_project.py b/rdmo/projects/tests/test_view_project.py index b43660cf14..f4e0028e36 100644 --- a/rdmo/projects/tests/test_view_project.py +++ b/rdmo/projects/tests/test_view_project.py @@ -602,6 +602,7 @@ def test_project_delete_post(db, client, username, password, project_id): assert Project.objects.filter(pk=project_id).first() +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) def test_project_export_xml(db, client, files, username, password, project_id): diff --git a/rdmo/projects/tests/test_view_project_create_import.py b/rdmo/projects/tests/test_view_project_create_import.py index f8af896dd5..597820ca2d 100644 --- a/rdmo/projects/tests/test_view_project_create_import.py +++ b/rdmo/projects/tests/test_view_project_create_import.py @@ -183,6 +183,7 @@ def test_project_create_import_post_upload_file_forbidden(db, client, settings): assert response.status_code == 403 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) def test_project_create_import_post_import_file(db, settings, client, files, username, password): client.login(username=username, password=password) @@ -238,6 +239,7 @@ def test_project_create_import_post_import_file(db, settings, client, files, use assert response.url.startswith('/account/login/') +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) def test_project_create_import_post_import_file_cancel(db, settings, client, files, username, password): client.login(username=username, password=password) diff --git a/rdmo/projects/tests/test_view_project_update_import.py b/rdmo/projects/tests/test_view_project_update_import.py index d2da3816b3..4c658a429c 100644 --- a/rdmo/projects/tests/test_view_project_update_import.py +++ b/rdmo/projects/tests/test_view_project_update_import.py @@ -151,6 +151,7 @@ def test_project_update_import_post_upload_file_empty(db, client, username, pass assert response.url.startswith('/account/login/') +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) def test_project_update_import_post_import_file(db, settings, client, files, username, password, project_id): @@ -223,6 +224,7 @@ def test_project_update_import_post_import_file(db, settings, client, files, use assert response.url.startswith('/account/login/') +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) def test_project_update_import_post_import_file_cancel(db, settings, client, files, username, password, project_id): diff --git a/rdmo/projects/tests/test_view_snapshot.py b/rdmo/projects/tests/test_view_snapshot.py index a49b161d0f..4b2758f390 100644 --- a/rdmo/projects/tests/test_view_snapshot.py +++ b/rdmo/projects/tests/test_view_snapshot.py @@ -57,6 +57,7 @@ def test_snapshot_create_get(db, client, username, password, project_id): assert response.status_code == 302 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) def test_snapshot_create_post(db, client, files, username, password, project_id): @@ -182,6 +183,7 @@ def test_snapshot_rollback_get(db, client, username, password, project_id, snaps assert response.status_code == 404 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) @pytest.mark.parametrize('snapshot_id', snapshots) diff --git a/rdmo/projects/tests/test_viewset_project_snapshot.py b/rdmo/projects/tests/test_viewset_project_snapshot.py index 2c3c7c7337..c95439f8cb 100644 --- a/rdmo/projects/tests/test_viewset_project_snapshot.py +++ b/rdmo/projects/tests/test_viewset_project_snapshot.py @@ -86,6 +86,7 @@ def test_detail(db, client, username, password, project_id, snapshot_id): assert response.status_code == 404 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) def test_create(db, client, files, username, password, project_id): diff --git a/rdmo/projects/tests/test_viewset_project_value.py b/rdmo/projects/tests/test_viewset_project_value.py index 0b5afb6ed0..8ef47a3544 100644 --- a/rdmo/projects/tests/test_viewset_project_value.py +++ b/rdmo/projects/tests/test_viewset_project_value.py @@ -266,6 +266,7 @@ def test_set(db, client, username, password, project_id, value_id, set_values_co assert Value.objects.count() == values_count +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) @pytest.mark.parametrize('value_id', values) @@ -285,6 +286,7 @@ def test_file_get(db, client, files, username, password, project_id, value_id): assert response.status_code == 404 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) @pytest.mark.parametrize('value_id', values) diff --git a/rdmo/projects/tests/test_viewset_value.py b/rdmo/projects/tests/test_viewset_value.py index a2b3dfbf80..bb90e44c31 100644 --- a/rdmo/projects/tests/test_viewset_value.py +++ b/rdmo/projects/tests/test_viewset_value.py @@ -142,6 +142,7 @@ def test_delete(db, client, username, password, value_id): assert response.status_code == 401 +@pytest.mark.files() @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('value_id', values) def test_file(db, client, files, username, password, value_id):