From 5030d4127af92327287dca5e115d7f1b3c3db493 Mon Sep 17 00:00:00 2001 From: Liran BG Date: Mon, 26 Dec 2022 10:56:46 +0200 Subject: [PATCH] [CI] Send slack notifications for failing system tests (#2811) --- .github/workflows/system-tests-enterprise.yml | 2 + automation/system_test/prepare.py | 8 +++ tests/system/conftest.py | 61 +++++++++++++++++++ tests/system/env-template.yml | 3 + 4 files changed, 74 insertions(+) create mode 100644 tests/system/conftest.py diff --git a/.github/workflows/system-tests-enterprise.yml b/.github/workflows/system-tests-enterprise.yml index f2a581a9bb1..eb41250bdb1 100644 --- a/.github/workflows/system-tests-enterprise.yml +++ b/.github/workflows/system-tests-enterprise.yml @@ -200,6 +200,7 @@ jobs: "${{ steps.computed_params.outputs.iguazio_version }}" \ "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \ "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \ + "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \ --mlrun-commit "${{ steps.computed_params.outputs.mlrun_hash }}" \ --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \ --override-image-repo ${{ steps.computed_params.outputs.mlrun_docker_repo }} \ @@ -247,6 +248,7 @@ jobs: "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \ "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \ "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \ + "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \ - name: Run System Tests run: | MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunSystemTestsCleanResources }}" \ diff --git a/automation/system_test/prepare.py b/automation/system_test/prepare.py index 9a53bbe8374..187d3c5dcaa 100644 --- a/automation/system_test/prepare.py +++ b/automation/system_test/prepare.py @@ -65,6 +65,7 @@ def __init__( iguazio_version: str = None, spark_service: str = None, password: str = None, + slack_webhook_url: str = None, debug: bool = False, ): self._logger = logger @@ -92,6 +93,7 @@ def __init__( "V3IO_USERNAME": username, "V3IO_ACCESS_KEY": access_key, "MLRUN_SYSTEM_TESTS_DEFAULT_SPARK_SERVICE": spark_service, + "MLRUN_SYSTEM_TESTS_SLACK_WEBHOOK_URL": slack_webhook_url, } if password: self._env_config["V3IO_PASSWORD"] = password @@ -542,6 +544,7 @@ def main(): @click.argument("iguazio-version", type=str, default=None, required=True) @click.argument("spark-service", type=str, required=True) @click.argument("password", type=str, default=None, required=False) +@click.argument("slack-webhook-url", type=str, default=None, required=False) @click.option( "--debug", "-d", @@ -567,6 +570,7 @@ def run( iguazio_version: str, spark_service: str, password: str, + slack_webhook_url: str, debug: bool, ): system_test_preparer = SystemTestPreparer( @@ -588,6 +592,7 @@ def run( iguazio_version, spark_service, password, + slack_webhook_url, debug, ) try: @@ -605,6 +610,7 @@ def run( @click.argument("access-key", type=str, required=True) @click.argument("spark-service", type=str, required=True) @click.argument("password", type=str, default=None, required=False) +@click.argument("slack-webhook-url", type=str, default=None, required=False) @click.option( "--debug", "-d", @@ -619,6 +625,7 @@ def env( access_key: str, spark_service: str, password: str, + slack_webhook_url: str, debug: bool, ): system_test_preparer = SystemTestPreparer( @@ -640,6 +647,7 @@ def env( spark_service=spark_service, password=password, debug=debug, + slack_webhook_url=slack_webhook_url, ) try: system_test_preparer.prepare_local_env() diff --git a/tests/system/conftest.py b/tests/system/conftest.py new file mode 100644 index 00000000000..a7949f8272b --- /dev/null +++ b/tests/system/conftest.py @@ -0,0 +1,61 @@ +# Copyright 2018 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os + +import pytest +import requests +from _pytest.python import Function +from _pytest.reports import TestReport +from _pytest.runner import CallInfo + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item: Function, call: CallInfo) -> TestReport: + outcome = yield + result: TestReport = outcome.get_result() + + try: + post_report_to_slack(result, os.getenv("MLRUN_SYSTEM_TESTS_SLACK_WEBHOOK_URL")) + except Exception as exc: + print(f"Failed to post test report to slack: {exc}") + + +def post_report_to_slack(report: TestReport, slack_webhook_url: str): + if not report.failed: + return + + if not slack_webhook_url: + return + + data = { + "text": f"Test `{report.head_line}` has failed.", + "attachments": [ + { + "color": "danger", + "title": "System test failure details", + "fields": [ + { + "title": "Failure", + "value": f"```{report.longreprtext}```", + "short": False, + }, + ], + }, + ], + } + res = requests.post(slack_webhook_url, json=data) + res.raise_for_status() diff --git a/tests/system/env-template.yml b/tests/system/env-template.yml index 75770a0da3c..70d728c6372 100644 --- a/tests/system/env-template.yml +++ b/tests/system/env-template.yml @@ -41,3 +41,6 @@ MLRUN_SYSTEM_TESTS_GOOGLE_BIG_QUERY_CREDENTIALS_JSON: # kafka brokers string for kafka tests MLRUN_SYSTEM_TESTS_KAFKA_BROKERS: + +# slack webhook for sending system test reports +MLRUN_SYSTEM_TESTS_SLACK_WEBHOOK_URL: