Fix Integration Testing Workflow (#14)

* Fix indentation

* Add logic to handle exit codes for timeout

* Capture symlinks db in artifacts

* Use tee to redirect to file

* Create output dirs before running simulator

* Adjust timeouts

* Adjust workflow

* Add badge to README

* Can't collect symlinks in artifacts

* Update projects config

* Generate debug logs

.github/workflows/integration_tests.yml

@@ -18,16 +18,19 @@ jobs:
       run: |
         mkdir -p artifacts
     - name: Setup Java
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '11'
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: '11'
     - name: Download Run Simulator
       run: |
         wget https://github.com/dfornika/illumina-run-simulator/releases/download/v0.1.0/illumina-run-simulator-0.1.0-standalone.jar -O illumina-run-simulator.jar
+    - name: Create simulated run output directories
+      run: |
+        mkdir -p artifacts/simulated_runs/{M00123,M00456,VH00123}/22
     - name: Run Run Simulator
       run: |
-        timeout 10 java -jar illumina-run-simulator.jar --config test/data/illumina-run-simulator/config.edn 2> >(tee artifacts/illumina-run-simulator.log.jsonl)
+        timeout 10 java -jar illumina-run-simulator.jar --config test/data/illumina-run-simulator/config.edn 2> >(tee artifacts/illumina-run-simulator.log.jsonl) || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
     - name: Setup Python
       uses: actions/setup-python@v4
       with:
@@ -41,12 +44,16 @@ jobs:
         mkdir alembic/versions
         alembic revision --autogenerate -m 'init'
         alembic upgrade head
+    - name: Create symlink output directories
+      run: |
+        mkdir -p test_output/symlinks_by_project
     - name: Run Auto Symlinker
       run: |
-        timeout 30 auto-fastq-symlink --config test/data/config/integration_testing/config.json 2> >(artifacts/auto-fastq-symlink.log.jsonl)
+        timeout 15 auto-fastq-symlink --config test/data/config/integration_testing/config.json --log-level debug 2> >(tee artifacts/auto-fastq-symlink.log.jsonl) || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+    - name: Copy db to Artifacts
+      run: |
+        cp symlinks.db artifacts
     - uses: actions/upload-artifact@v1
       with:
         name: Artifacts
         path: artifacts
-
-

.github/workflows/unit_tests.yml

@@ -12,8 +12,10 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - uses: actions/setup-python@v4
+    - name: Checkout Repo
+      uses: actions/checkout@v1
+    - name: Setup Python
+      uses: actions/setup-python@v4
       with:
         python-version: '3.10'
     - name: Create artifacts directory

README.md

@@ -1,4 +1,5 @@
 [![Unit Tests](https://github.com/BCCDC-PHL/auto-fastq-symlink/actions/workflows/unit_tests.yml/badge.svg?branch=main)](https://github.com/BCCDC-PHL/auto-fastq-symlink/actions/workflows/unit_tests.yml)
+[![Integration Tests](https://github.com/BCCDC-PHL/auto-fastq-symlink/actions/workflows/integration_tests.yml/badge.svg?branch=main)](https://github.com/BCCDC-PHL/auto-fastq-symlink/actions/workflows/integration_tests.yml)
 
 # auto-fastq-symlink
 Automated symlinking of illumina fastq files, organized by project.

test/data/config/integration_testing/projects.csv

@@ -1,4 +1,4 @@
 project_id,fastq_symlinks_dir,excluded_runs_list,excluded_libraries_list,simplify_symlink_filenames
-assay_development,artifacts/symlinks_by_project/assay_development,test/data/config/integration_testing/assay_development_excluded_runs.csv,test/data/config/integration_testing/assay_development_excluded_libraries.csv,False
-viral_outbreak,artifacts/symlinks_by_project/viral_outbreak,test/data/config/integration_testing/viral_outbreak_excluded_runs.csv,test/data/config/integration_testing/viral_outbreak_excluded_libraries.csv,True
-antibiotic_resistance,artifacts/symlinks_by_project/antibiotic_resistance,test/data/config/integration_testing/antibiotic_resistance_excluded_runs.csv,test/data/config/integration_testing/antibiotic_resistance_excluded_libraries.csv,True
+assay_development,test_output/symlinks_by_project/assay_development,test/data/config/integration_testing/assay_development_excluded_runs.csv,test/data/config/integration_testing/assay_development_excluded_libraries.csv,False
+viral_outbreak,test_output/symlinks_by_project/viral_outbreak,test/data/config/integration_testing/viral_outbreak_excluded_runs.csv,test/data/config/integration_testing/viral_outbreak_excluded_libraries.csv,True
+antibiotic_resistance,test_output/symlinks_by_project/antibiotic_resistance,test/data/config/integration_testing/antibiotic_resistance_excluded_runs.csv,test/data/config/integration_testing/antibiotic_resistance_excluded_libraries.csv,True