[Development] Retain cause exceptions in logs and raised errors (mlru…

…n#2795)

mlrun/__main__.py

@@ -36,6 +36,7 @@
 from .builder import upload_tarball
 from .config import config as mlconf
 from .db import get_run_db
+from .errors import err_to_str
 from .k8s_utils import K8sHelper
 from .model import RunTemplate
 from .platforms import auto_mount as auto_mount_modifier
@@ -412,7 +413,7 @@ def run(
         if resp and dump:
             print(resp.to_yaml())
     except RunError as err:
-        print(f"runtime error: {err}")
+        print(f"runtime error: {err_to_str(err)}")
         exit(1)
 
 
@@ -553,7 +554,7 @@ def build(
                 with_mlrun=with_mlrun, watch=not silent, is_kfp=kfp, skip_deployed=skip
             )
         except Exception as err:
-            print(f"deploy error, {err}")
+            print(f"deploy error, {err_to_str(err)}")
             exit(1)
 
         state = func.status.state
@@ -666,7 +667,7 @@ def deploy(
     try:
         addr = function.deploy(dashboard=dashboard, project=project, tag=tag)
     except Exception as err:
-        print(f"deploy error: {err}")
+        print(f"deploy error: {err_to_str(err)}")
         exit(1)
 
     print(f"function deployed, address={addr}")
@@ -1087,7 +1088,7 @@ def project(
             )
         except Exception as exc:
             print(traceback.format_exc())
-            message = f"failed to run pipeline, {exc}"
+            message = f"failed to run pipeline, {err_to_str(exc)}"
             had_error = True
             print(message)
 
@@ -1360,7 +1361,7 @@ def func_url_to_runtime(func_url, ensure_project: bool = False):
                 function.spec.command = command
             runtime = function.to_dict()
     except Exception as exc:
-        logger.error(f"function {func_url} not found, {exc}")
+        logger.error(f"function {func_url} not found, {err_to_str(exc)}")
         return None
 
     if not runtime:

mlrun/api/api/endpoints/files.py

@@ -23,6 +23,7 @@
 import mlrun.api.utils.auth.verifier
 from mlrun.api.api.utils import get_obj_path, get_secrets, log_and_raise
 from mlrun.datastore import store_manager
+from mlrun.errors import err_to_str
 from mlrun.utils import logger
 
 router = fastapi.APIRouter()
@@ -139,7 +140,7 @@ def _get_files(
 
         body = obj.get(size, offset)
     except FileNotFoundError as exc:
-        log_and_raise(HTTPStatus.NOT_FOUND.value, path=objpath, err=str(exc))
+        log_and_raise(HTTPStatus.NOT_FOUND.value, path=objpath, err=err_to_str(exc))
 
     if body is None:
         log_and_raise(HTTPStatus.NOT_FOUND.value, path=objpath)
@@ -176,7 +177,7 @@ def _get_filestat(
     try:
         stat = store_manager.object(url=path, secrets=secrets).stat()
     except FileNotFoundError as exc:
-        log_and_raise(HTTPStatus.NOT_FOUND.value, path=path, err=str(exc))
+        log_and_raise(HTTPStatus.NOT_FOUND.value, path=path, err=err_to_str(exc))
 
     ctype, _ = mimetypes.guess_type(path)
     if not ctype:

mlrun/api/api/endpoints/functions.py

@@ -47,7 +47,7 @@
 from mlrun.api.utils.singletons.k8s import get_k8s
 from mlrun.builder import build_runtime
 from mlrun.config import config
-from mlrun.errors import MLRunRuntimeError
+from mlrun.errors import MLRunRuntimeError, err_to_str
 from mlrun.run import new_function
 from mlrun.runtimes import RuntimeKinds, ServingRuntime, runtime_resources_map
 from mlrun.runtimes.function import deploy_nuclio_function, get_nuclio_deploy_status
@@ -514,7 +514,10 @@ def _build_function(
         fn = new_function(runtime=function)
     except Exception as err:
         logger.error(traceback.format_exc())
-        log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}")
+        log_and_raise(
+            HTTPStatus.BAD_REQUEST.value,
+            reason=f"runtime error: {err_to_str(err)}",
+        )
     try:
         run_db = get_run_db_instance(db_session)
         fn.set_db_connection(run_db)
@@ -587,7 +590,10 @@ def _build_function(
         logger.info("Fn:\n %s", fn.to_yaml())
     except Exception as err:
         logger.error(traceback.format_exc())
-        log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}")
+        log_and_raise(
+            HTTPStatus.BAD_REQUEST.value,
+            reason=f"runtime error: {err_to_str(err)}",
+        )
     return fn, ready
 
 
@@ -637,7 +643,10 @@ def _start_function(
             logger.info("Fn:\n %s", function.to_yaml())
         except Exception as err:
             logger.error(traceback.format_exc())
-            log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}")
+            log_and_raise(
+                HTTPStatus.BAD_REQUEST.value,
+                reason=f"runtime error: {err_to_str(err)}",
+            )
     finally:
         mlrun.api.db.session.close_session(db_session)
 
@@ -678,7 +687,10 @@ def _get_function_status(data, auth_info: mlrun.api.schemas.AuthInfo):
         logger.info("status: %s", resp)
     except Exception as err:
         logger.error(traceback.format_exc())
-        log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}")
+        log_and_raise(
+            HTTPStatus.BAD_REQUEST.value,
+            reason=f"runtime error: {err_to_str(err)}",
+        )
 
 
 def _create_model_monitoring_stream(project: str):

mlrun/api/api/utils.py

@@ -40,6 +40,7 @@
 from mlrun.api.utils.singletons.scheduler import get_scheduler
 from mlrun.config import config
 from mlrun.db.sqldb import SQLDB as SQLRunDB
+from mlrun.errors import err_to_str
 from mlrun.k8s_utils import get_k8s_helper
 from mlrun.run import import_function, new_function
 from mlrun.runtimes.utils import enrich_function_from_dict
@@ -714,7 +715,10 @@ def _submit_run(
         raise
     except Exception as err:
         logger.error(traceback.format_exc())
-        log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}")
+        log_and_raise(
+            HTTPStatus.BAD_REQUEST.value,
+            reason=f"runtime error: {err_to_str(err)}",
+        )
 
     logger.info("Run submission succeeded", response=response)
     return project, fn.kind, run_uid, {"data": response}

mlrun/api/crud/pipelines.py

@@ -29,6 +29,7 @@
 import mlrun.kfpops
 import mlrun.utils.helpers
 import mlrun.utils.singleton
+from mlrun.errors import err_to_str
 from mlrun.utils import logger
 
 
@@ -130,7 +131,7 @@ def get_pipeline(
 
         except Exception as exc:
             raise mlrun.errors.MLRunRuntimeError(
-                f"Failed getting kfp run: {exc}"
+                f"Failed getting kfp run: {err_to_str(exc)}"
             ) from exc
 
         return run
@@ -185,9 +186,11 @@ def create_pipeline(
             logger.warning(
                 "Failed creating pipeline",
                 traceback=traceback.format_exc(),
-                exc=str(exc),
+                exc=err_to_str(exc),
+            )
+            raise mlrun.errors.MLRunBadRequestError(
+                f"Failed creating pipeline: {err_to_str(exc)}"
             )
-            raise mlrun.errors.MLRunBadRequestError(f"Failed creating pipeline: {exc}")
         finally:
             pipeline_file.close()
 

mlrun/api/db/sqldb/db.py

@@ -61,6 +61,7 @@
     _tagged,
 )
 from mlrun.config import config
+from mlrun.errors import err_to_str
 from mlrun.lists import ArtifactList, FunctionList, RunList
 from mlrun.model import RunObject
 from mlrun.utils import (
@@ -107,7 +108,9 @@ def _try_function():
                 if mlrun.utils.helpers.are_strings_in_exception_chain_messages(
                     exc, conflict_messages
                 ):
-                    logger.warning("Got conflict error from DB. Retrying", err=str(exc))
+                    logger.warning(
+                        "Got conflict error from DB. Retrying", err=err_to_str(exc)
+                    )
                     raise mlrun.errors.MLRunRuntimeError(
                         "Got conflict error from DB"
                     ) from exc
@@ -2640,7 +2643,9 @@ def _find_or_create_users(self, session, user_names):
                 session.commit()
             except SQLAlchemyError as err:
                 session.rollback()
-                raise mlrun.errors.MLRunConflictError(f"add user: {err}") from err
+                raise mlrun.errors.MLRunConflictError(
+                    f"add user: {err_to_str(err)}"
+                ) from err
         return users
 
     def _get_class_instance_by_uid(self, session, cls, name, project, uid):
@@ -2690,7 +2695,9 @@ def _try_commit_obj():
                     raise mlrun.errors.MLRunRuntimeError(
                         "Failed committing changes, database is locked"
                     ) from err
-                logger.warning("Failed committing changes to DB", cls=cls, err=str(err))
+                logger.warning(
+                    "Failed committing changes to DB", cls=cls, err=err_to_str(err)
+                )
                 if not ignore:
                     identifiers = ",".join(
                         object_.get_identifier_string() for object_ in objects

mlrun/api/initial_data.py

@@ -35,6 +35,7 @@
 from mlrun.api.db.init_db import init_db
 from mlrun.api.db.session import close_session, create_session
 from mlrun.config import config
+from mlrun.errors import err_to_str
 from mlrun.utils import is_legacy_artifact, logger
 
 
@@ -569,7 +570,7 @@ def _resolve_current_data_version(
         ):
             logger.info(
                 "Data version table does not exist, assuming prior version",
-                exc=exc,
+                exc=err_to_str(exc),
                 data_version_prior_to_table_addition=data_version_prior_to_table_addition,
             )
             return data_version_prior_to_table_addition

mlrun/api/main.py

@@ -45,6 +45,7 @@
 )
 from mlrun.api.utils.singletons.scheduler import get_scheduler, initialize_scheduler
 from mlrun.config import config
+from mlrun.errors import err_to_str
 from mlrun.k8s_utils import get_k8s_helper
 from mlrun.runtimes import RuntimeKinds, get_runtime_handler
 from mlrun.utils import logger
@@ -269,7 +270,7 @@ async def _synchronize_with_chief_clusterization_spec():
     except Exception as exc:
         logger.debug(
             "Failed receiving clusterization spec",
-            exc=str(exc),
+            exc=err_to_str(exc),
             traceback=traceback.format_exc(),
         )
     else:
@@ -319,7 +320,9 @@ def _monitor_runs():
                 runtime_handler.monitor_runs(get_db(), db_session)
             except Exception as exc:
                 logger.warning(
-                    "Failed monitoring runs. Ignoring", exc=str(exc), kind=kind
+                    "Failed monitoring runs. Ignoring",
+                    exc=err_to_str(exc),
+                    kind=kind,
                 )
     finally:
         close_session(db_session)
@@ -334,7 +337,9 @@ def _cleanup_runtimes():
                 runtime_handler.delete_resources(get_db(), db_session)
             except Exception as exc:
                 logger.warning(
-                    "Failed deleting resources. Ignoring", exc=str(exc), kind=kind
+                    "Failed deleting resources. Ignoring",
+                    exc=err_to_str(exc),
+                    kind=kind,
                 )
     finally:
         close_session(db_session)

mlrun/api/utils/projects/follower.py

@@ -37,6 +37,7 @@
 import mlrun.utils.helpers
 import mlrun.utils.regex
 import mlrun.utils.singleton
+from mlrun.errors import err_to_str
 from mlrun.utils import logger
 
 
@@ -83,7 +84,7 @@ def initialize(self):
         except Exception as exc:
             logger.warning(
                 "Initial projects sync failed",
-                exc=str(exc),
+                exc=err_to_str(exc),
                 traceback=traceback.format_exc(),
             )
         self._start_periodic_sync()

mlrun/api/utils/projects/leader.py

@@ -32,6 +32,7 @@
 import mlrun.utils.helpers
 import mlrun.utils.regex
 import mlrun.utils.singleton
+from mlrun.errors import err_to_str
 from mlrun.utils import logger
 
 
@@ -273,7 +274,7 @@ def _ensure_project_synced(
                     project_follower_name=project_follower_name,
                     project=project,
                     project_name=project_name,
-                    exc=str(exc),
+                    exc=err_to_str(exc),
                     traceback=traceback.format_exc(),
                 )
             else:
@@ -329,7 +330,7 @@ def _store_project_in_followers(
                     follower_name=follower_name,
                     project_name=project_name,
                     project=project,
-                    exc=str(exc),
+                    exc=err_to_str(exc),
                     traceback=traceback.format_exc(),
                 )
 
@@ -363,7 +364,7 @@ def _create_project_in_missing_followers(
                     project_follower_name=project_follower_name,
                     project_name=project_name,
                     project=project,
-                    exc=str(exc),
+                    exc=err_to_str(exc),
                     traceback=traceback.format_exc(),
                 )
 

mlrun/api/utils/scheduler.py

@@ -34,6 +34,7 @@
 from mlrun.api.db.session import close_session, create_session
 from mlrun.api.utils.singletons.db import get_db
 from mlrun.config import config
+from mlrun.errors import err_to_str
 from mlrun.model import RunObject
 from mlrun.runtimes.constants import RunStates
 from mlrun.utils import logger
@@ -711,7 +712,7 @@ def _reload_schedules(self, db_session: Session):
             except Exception as exc:
                 logger.warn(
                     "Failed rescheduling job. Continuing",
-                    exc=str(exc),
+                    exc=err_to_str(exc),
                     traceback=traceback.format_exc(),
                     db_schedule=db_schedule,
                 )

mlrun/config.py

@@ -38,6 +38,7 @@
 import yaml
 
 import mlrun.errors
+from mlrun.errors import err_to_str
 
 env_prefix = "MLRUN_"
 env_file_key = f"{env_prefix}CONFIG_FILE"
@@ -518,7 +519,9 @@ def update(self, cfg, skip_errors=False):
                     except mlrun.errors.MLRunRuntimeError as exc:
                         if not skip_errors:
                             raise exc
-                        print(f"Warning, failed to set config key {key}={value}, {exc}")
+                        print(
+                            f"Warning, failed to set config key {key}={value}, {err_to_str(exc)}"
+                        )
 
     def dump_yaml(self, stream=None):
         return yaml.dump(self._cfg, stream, default_flow_style=False)

mlrun/datastore/azure_blob.py

@@ -19,6 +19,7 @@
 from azure.storage.blob import BlobServiceClient
 
 import mlrun.errors
+from mlrun.errors import err_to_str
 
 from .base import DataStore, FileStats
 
@@ -46,7 +47,7 @@ def get_filesystem(self, silent=True):
         except ImportError as exc:
             if not silent:
                 raise ImportError(
-                    f"Azure adlfs not installed, run pip install adlfs, {exc}"
+                    f"Azure adlfs not installed, run pip install adlfs, {err_to_str(exc)}"
                 )
             return None
         self._filesystem = fsspec.filesystem(self.kind, **self.get_storage_options())