[CI] Send slack notifications for failing system tests (mlrun#2811)

.github/workflows/system-tests-enterprise.yml

@@ -200,6 +200,7 @@ jobs:
           "${{ steps.computed_params.outputs.iguazio_version }}" \
           "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
           "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \
+          "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
           --mlrun-commit "${{ steps.computed_params.outputs.mlrun_hash }}" \
           --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
           --override-image-repo ${{ steps.computed_params.outputs.mlrun_docker_repo }} \
@@ -247,6 +248,7 @@ jobs:
           "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
           "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
           "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \
+          "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
     - name: Run System Tests
       run: |
         MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunSystemTestsCleanResources }}" \

automation/system_test/prepare.py

@@ -65,6 +65,7 @@ def __init__(
         iguazio_version: str = None,
         spark_service: str = None,
         password: str = None,
+        slack_webhook_url: str = None,
         debug: bool = False,
     ):
         self._logger = logger
@@ -92,6 +93,7 @@ def __init__(
             "V3IO_USERNAME": username,
             "V3IO_ACCESS_KEY": access_key,
             "MLRUN_SYSTEM_TESTS_DEFAULT_SPARK_SERVICE": spark_service,
+            "MLRUN_SYSTEM_TESTS_SLACK_WEBHOOK_URL": slack_webhook_url,
         }
         if password:
             self._env_config["V3IO_PASSWORD"] = password
@@ -542,6 +544,7 @@ def main():
 @click.argument("iguazio-version", type=str, default=None, required=True)
 @click.argument("spark-service", type=str, required=True)
 @click.argument("password", type=str, default=None, required=False)
+@click.argument("slack-webhook-url", type=str, default=None, required=False)
 @click.option(
     "--debug",
     "-d",
@@ -567,6 +570,7 @@ def run(
     iguazio_version: str,
     spark_service: str,
     password: str,
+    slack_webhook_url: str,
     debug: bool,
 ):
     system_test_preparer = SystemTestPreparer(
@@ -588,6 +592,7 @@ def run(
         iguazio_version,
         spark_service,
         password,
+        slack_webhook_url,
         debug,
     )
     try:
@@ -605,6 +610,7 @@ def run(
 @click.argument("access-key", type=str, required=True)
 @click.argument("spark-service", type=str, required=True)
 @click.argument("password", type=str, default=None, required=False)
+@click.argument("slack-webhook-url", type=str, default=None, required=False)
 @click.option(
     "--debug",
     "-d",
@@ -619,6 +625,7 @@ def env(
     access_key: str,
     spark_service: str,
     password: str,
+    slack_webhook_url: str,
     debug: bool,
 ):
     system_test_preparer = SystemTestPreparer(
@@ -640,6 +647,7 @@ def env(
         spark_service=spark_service,
         password=password,
         debug=debug,
+        slack_webhook_url=slack_webhook_url,
     )
     try:
         system_test_preparer.prepare_local_env()

tests/system/conftest.py

@@ -0,0 +1,61 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import os
+
+import pytest
+import requests
+from _pytest.python import Function
+from _pytest.reports import TestReport
+from _pytest.runner import CallInfo
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item: Function, call: CallInfo) -> TestReport:
+    outcome = yield
+    result: TestReport = outcome.get_result()
+
+    try:
+        post_report_to_slack(result, os.getenv("MLRUN_SYSTEM_TESTS_SLACK_WEBHOOK_URL"))
+    except Exception as exc:
+        print(f"Failed to post test report to slack: {exc}")
+
+
+def post_report_to_slack(report: TestReport, slack_webhook_url: str):
+    if not report.failed:
+        return
+
+    if not slack_webhook_url:
+        return
+
+    data = {
+        "text": f"Test `{report.head_line}` has failed.",
+        "attachments": [
+            {
+                "color": "danger",
+                "title": "System test failure details",
+                "fields": [
+                    {
+                        "title": "Failure",
+                        "value": f"```{report.longreprtext}```",
+                        "short": False,
+                    },
+                ],
+            },
+        ],
+    }
+    res = requests.post(slack_webhook_url, json=data)
+    res.raise_for_status()

tests/system/env-template.yml

@@ -41,3 +41,6 @@ MLRUN_SYSTEM_TESTS_GOOGLE_BIG_QUERY_CREDENTIALS_JSON:
 
 # kafka brokers string for kafka tests
 MLRUN_SYSTEM_TESTS_KAFKA_BROKERS:
+
+# slack webhook for sending system test reports
+MLRUN_SYSTEM_TESTS_SLACK_WEBHOOK_URL: