feat(ci): integrated tracer tests as a cron job to store the results …

…in a separate field in the database (ivy-llc#27872)
mosesdaudu001 · Jan 8, 2024 · 314106b · 314106b
1 parent 0676f59
commit 314106b
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 14 deletions.
diff --git a/.github/workflows/intelligent-tests.yml b/.github/workflows/intelligent-tests.yml
@@ -89,7 +89,7 @@ jobs:
         run: |
           cd ivy
           set -o pipefail
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt
         continue-on-error: true
 
       - name: Upload test results

diff --git a/.github/workflows/manual-tests.yml b/.github/workflows/manual-tests.yml
@@ -17,6 +17,11 @@ on:
         default: false
         required: false
 
+      tracer:
+        description: 'Tracer Testing :'
+        default: false
+        required: false
+
 permissions:
   actions: read
 
@@ -59,7 +64,7 @@ jobs:
           pip3 install pymongo
           cd ivy
           python3 scripts/setup_tests/setup_tests.py ${{ github.event.inputs.test }}
-          python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' ${{ github.event.inputs.gpu }} ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }}
+          python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' ${{ github.event.inputs.gpu }} ${{ github.run_id }} 'false' ${{ github.event.inputs.tracer }} ${{ steps.jobs.outputs.html_url }}
         continue-on-error: true
 
       - name: Check on failures
@@ -101,7 +106,7 @@ jobs:
           cd ivy
           sudo pip3 install -e .
           python scripts/setup_tests/setup_tests.py "${{ github.event.inputs.test }}"
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} ${{ github.event.inputs.version}} 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }}
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} ${{ github.event.inputs.version}} 'false' ${{ github.run_id }} 'false' ${{ github.event.inputs.tracer }} ${{ steps.jobs.outputs.html_url }}
         continue-on-error: true
 
       - name: Check on failures

diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -35,9 +35,9 @@ jobs:
       - name: Run CPU Tests
         run: |
           cd ivy
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'true' ${{ steps.jobs.outputs.html_url }}
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'true' 'false' ${{ steps.jobs.outputs.html_url }}
 
       - name: Run GPU Tests
         run: |
           cd ivy
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'true' ${{ steps.jobs.outputs.html_url }}
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'true' 'false' ${{ steps.jobs.outputs.html_url }}
diff --git a/.github/workflows/run-all-tests.yml b/.github/workflows/run-all-tests.yml
@@ -64,7 +64,7 @@ jobs:
           cd ivy
           python scripts/setup_tests/filter_tests.py ${{ matrix.branch }}
           set -o pipefail
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt
         continue-on-error: true
 
       - name: Upload test results

diff --git a/.github/workflows/test-ivy-cron-gpu.yml b/.github/workflows/test-ivy-cron-gpu.yml
@@ -75,7 +75,7 @@ jobs:
           pip3 install pymongo
           cd ivy
           python3 scripts/setup_tests/cron_tests.py ${{ github.run_number }} true
-          python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }}
+          python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }}
 
   stop-vm:
     needs: run-gpu-tests

diff --git a/.github/workflows/test-ivy-cron-multi-version.yml b/.github/workflows/test-ivy-cron-multi-version.yml
@@ -38,7 +38,7 @@ jobs:
           cd ivy
           pip3 install pymongo
           python scripts/setup_tests/cron_tests_multi_version.py ${{ github.run_number }}
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'true' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }}
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'true' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }}
         continue-on-error: true
 
       - name: Check on failures

diff --git a/.github/workflows/test-ivy-cron.yml b/.github/workflows/test-ivy-cron.yml
@@ -38,7 +38,7 @@ jobs:
           cd ivy
           pip3 install pymongo
           python scripts/setup_tests/cron_tests.py ${{ github.run_number }} false
-          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }}
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }}
         continue-on-error: true
 
       - name: Check on failures

diff --git a/.github/workflows/test-ivy-tracer-cron.yml b/.github/workflows/test-ivy-tracer-cron.yml
@@ -0,0 +1,46 @@
+name: test-ivy-cron
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '15 * * * *'
+permissions:
+  actions: read
+jobs:
+  run-nightly-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️Ivy
+        uses: actions/checkout@v2
+        with:
+          path: ivy
+          persist-credentials: false
+          submodules: "recursive"
+
+      - name: Install ivy and fetch binaries
+        run: |
+          cd ivy
+          sudo pip3 install -e .
+          mkdir .ivy
+          touch .ivy/key.pem
+          echo -n ${{ secrets.USER_API_KEY }} > .ivy/key.pem
+          cd ..
+
+      - name: Get Job URL
+        uses: Tiryoh/gha-jobid-action@v0
+        id: jobs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: ${{ github.job }}
+
+      - name: Run Tests
+        id: tests
+        run: |
+          cd ivy
+          pip3 install pymongo
+          python scripts/setup_tests/cron_tests.py ${{ github.run_number }} false
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'true' ${{ steps.jobs.outputs.html_url }}
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.tests.outcome != 'success'
+        run: exit 1
diff --git a/scripts/run_tests/run_tests.py b/scripts/run_tests/run_tests.py
@@ -20,16 +20,24 @@
     gpu_flag = sys.argv[5]
     workflow_id = sys.argv[6]
     priority_flag = sys.argv[7]
+    tracer_flag = sys.argv[8]
 
-    if len(sys.argv) > 8 and sys.argv[8] != "null":
-        run_id = sys.argv[8]
+    if len(sys.argv) > 9 and sys.argv[9] != "null":
+        run_id = sys.argv[9]
     else:
         run_id = f"https://github.com/unifyai/ivy/actions/runs/{workflow_id}"
 
     device = "cpu"
     if gpu_flag == "true":
         device = "gpu"
 
+    tracer_str = ""
+    if tracer_flag == "true":
+        tracer_flag = "tracer_"
+        tracer_str = " --with-trace-testing"
+    else:
+        tracer_flag = ""
+
     cluster = MongoClient(
         f"mongodb+srv://deep-ivy:{mongo_key}@cluster0.qdvf8q3.mongodb.net/?retryWrites=true&w=majority"  # noqa
     )
@@ -111,7 +119,7 @@
                 )
                 command = (
                     "docker exec test-container python3 -m pytest --tb=short"
-                    f" {test_path}{device_str} --backend {backend}"
+                    f" {test_path}{device_str} --backend {backend}{tracer_str}"
                 )
                 os.system(command)
 
@@ -207,8 +215,12 @@
                 "_id": function_name,
                 "test_path": test_path,
                 "submodule": submodule,
-                f"{prefix_str}{backend}.{version}.status.{device}": not failed,
-                f"{prefix_str}{backend}.{version}.workflow.{device}": run_id,
+                f"{prefix_str}{backend}.{version}.{tracer_flag}status.{device}": (
+                    not failed
+                ),
+                f"{prefix_str}{backend}.{version}.{tracer_flag}workflow.{device}": (
+                    run_id
+                ),
             }
 
             # add transpilation metrics if report generated