diff --git a/.github/workflows/intelligent-tests.yml b/.github/workflows/intelligent-tests.yml index d4e2ec5de132a..6614e53eb42fd 100644 --- a/.github/workflows/intelligent-tests.yml +++ b/.github/workflows/intelligent-tests.yml @@ -89,7 +89,7 @@ jobs: run: | cd ivy set -o pipefail - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt continue-on-error: true - name: Upload test results diff --git a/.github/workflows/manual-tests.yml b/.github/workflows/manual-tests.yml index 0fa60e5338106..71c231e952c49 100644 --- a/.github/workflows/manual-tests.yml +++ b/.github/workflows/manual-tests.yml @@ -17,6 +17,11 @@ on: default: false required: false + tracer: + description: 'Tracer Testing :' + default: false + required: false + permissions: actions: read @@ -59,7 +64,7 @@ jobs: pip3 install pymongo cd ivy python3 scripts/setup_tests/setup_tests.py ${{ github.event.inputs.test }} - python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' ${{ github.event.inputs.gpu }} ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} + python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' ${{ github.event.inputs.gpu }} ${{ github.run_id }} 'false' ${{ github.event.inputs.tracer }} ${{ steps.jobs.outputs.html_url }} continue-on-error: true - name: Check on failures @@ -101,7 +106,7 @@ jobs: cd ivy sudo pip3 install -e . python scripts/setup_tests/setup_tests.py "${{ github.event.inputs.test }}" - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} ${{ github.event.inputs.version}} 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} ${{ github.event.inputs.version}} 'false' ${{ github.run_id }} 'false' ${{ github.event.inputs.tracer }} ${{ steps.jobs.outputs.html_url }} continue-on-error: true - name: Check on failures diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index c7ccc919b6e61..249038c320383 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -35,9 +35,9 @@ jobs: - name: Run CPU Tests run: | cd ivy - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'true' ${{ steps.jobs.outputs.html_url }} + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'true' 'false' ${{ steps.jobs.outputs.html_url }} - name: Run GPU Tests run: | cd ivy - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'true' ${{ steps.jobs.outputs.html_url }} + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'true' 'false' ${{ steps.jobs.outputs.html_url }} diff --git a/.github/workflows/run-all-tests.yml b/.github/workflows/run-all-tests.yml index 5464f50283b52..c7bebee196cf6 100644 --- a/.github/workflows/run-all-tests.yml +++ b/.github/workflows/run-all-tests.yml @@ -64,7 +64,7 @@ jobs: cd ivy python scripts/setup_tests/filter_tests.py ${{ matrix.branch }} set -o pipefail - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt continue-on-error: true - name: Upload test results diff --git a/.github/workflows/test-ivy-cron-gpu.yml b/.github/workflows/test-ivy-cron-gpu.yml index 08ee34cd83dc1..c917cb74133b2 100644 --- a/.github/workflows/test-ivy-cron-gpu.yml +++ b/.github/workflows/test-ivy-cron-gpu.yml @@ -75,7 +75,7 @@ jobs: pip3 install pymongo cd ivy python3 scripts/setup_tests/cron_tests.py ${{ github.run_number }} true - python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} + python3 scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'true' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} stop-vm: needs: run-gpu-tests diff --git a/.github/workflows/test-ivy-cron-multi-version.yml b/.github/workflows/test-ivy-cron-multi-version.yml index 14d9e02820c75..5f4177b3d0cf7 100644 --- a/.github/workflows/test-ivy-cron-multi-version.yml +++ b/.github/workflows/test-ivy-cron-multi-version.yml @@ -38,7 +38,7 @@ jobs: cd ivy pip3 install pymongo python scripts/setup_tests/cron_tests_multi_version.py ${{ github.run_number }} - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'true' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'true' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} continue-on-error: true - name: Check on failures diff --git a/.github/workflows/test-ivy-cron.yml b/.github/workflows/test-ivy-cron.yml index 7ee07cd6d3b99..59790144c2d41 100644 --- a/.github/workflows/test-ivy-cron.yml +++ b/.github/workflows/test-ivy-cron.yml @@ -38,7 +38,7 @@ jobs: cd ivy pip3 install pymongo python scripts/setup_tests/cron_tests.py ${{ github.run_number }} false - python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' ${{ steps.jobs.outputs.html_url }} + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }} continue-on-error: true - name: Check on failures diff --git a/.github/workflows/test-ivy-tracer-cron.yml b/.github/workflows/test-ivy-tracer-cron.yml new file mode 100644 index 0000000000000..edbd67ddfdd45 --- /dev/null +++ b/.github/workflows/test-ivy-tracer-cron.yml @@ -0,0 +1,46 @@ +name: test-ivy-cron +on: + workflow_dispatch: + schedule: + - cron: '15 * * * *' +permissions: + actions: read +jobs: + run-nightly-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout 🛎️Ivy + uses: actions/checkout@v2 + with: + path: ivy + persist-credentials: false + submodules: "recursive" + + - name: Install ivy and fetch binaries + run: | + cd ivy + sudo pip3 install -e . + mkdir .ivy + touch .ivy/key.pem + echo -n ${{ secrets.USER_API_KEY }} > .ivy/key.pem + cd .. + + - name: Get Job URL + uses: Tiryoh/gha-jobid-action@v0 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: ${{ github.job }} + + - name: Run Tests + id: tests + run: | + cd ivy + pip3 install pymongo + python scripts/setup_tests/cron_tests.py ${{ github.run_number }} false + python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'true' ${{ steps.jobs.outputs.html_url }} + continue-on-error: true + + - name: Check on failures + if: steps.tests.outcome != 'success' + run: exit 1 diff --git a/scripts/run_tests/run_tests.py b/scripts/run_tests/run_tests.py index ac5704b461004..0df6d1d59f32f 100644 --- a/scripts/run_tests/run_tests.py +++ b/scripts/run_tests/run_tests.py @@ -20,9 +20,10 @@ gpu_flag = sys.argv[5] workflow_id = sys.argv[6] priority_flag = sys.argv[7] + tracer_flag = sys.argv[8] - if len(sys.argv) > 8 and sys.argv[8] != "null": - run_id = sys.argv[8] + if len(sys.argv) > 9 and sys.argv[9] != "null": + run_id = sys.argv[9] else: run_id = f"https://github.com/unifyai/ivy/actions/runs/{workflow_id}" @@ -30,6 +31,13 @@ if gpu_flag == "true": device = "gpu" + tracer_str = "" + if tracer_flag == "true": + tracer_flag = "tracer_" + tracer_str = " --with-trace-testing" + else: + tracer_flag = "" + cluster = MongoClient( f"mongodb+srv://deep-ivy:{mongo_key}@cluster0.qdvf8q3.mongodb.net/?retryWrites=true&w=majority" # noqa ) @@ -111,7 +119,7 @@ ) command = ( "docker exec test-container python3 -m pytest --tb=short" - f" {test_path}{device_str} --backend {backend}" + f" {test_path}{device_str} --backend {backend}{tracer_str}" ) os.system(command) @@ -207,8 +215,12 @@ "_id": function_name, "test_path": test_path, "submodule": submodule, - f"{prefix_str}{backend}.{version}.status.{device}": not failed, - f"{prefix_str}{backend}.{version}.workflow.{device}": run_id, + f"{prefix_str}{backend}.{version}.{tracer_flag}status.{device}": ( + not failed + ), + f"{prefix_str}{backend}.{version}.{tracer_flag}workflow.{device}": ( + run_id + ), } # add transpilation metrics if report generated