Merge remote-tracking branch 'upstream/development' into development

conda-arm64-requirements.txt

@@ -1,5 +1,4 @@
-# required for v3io client, see docs/requirements.txt for the constraint
+# Required for v3io-frames client through grpcio-tools:
+# https://github.com/v3io/frames/blob/5490b1e7f281a8c44ec679105957e181c3fa81fc/clients/py/requirements.txt#L4
+# grpcio-tools 1.48.2 requires protobuf<4.0dev,>=3.12.0
 protobuf>=3.20.3, <4
-
-# see requirements.txt for the constraint
-pyyaml>=5.4.1, <7

docs/install/remote.md

@@ -43,6 +43,14 @@ Run ```pip install mlrun```
 To install a specific version, use the command: `pip install mlrun==<version>`. Replace the `<version>` placeholder with the MLRun version number.
 ```
 
+## Note for ARM64 (Apple Silicon) Users
+
+When using ARM64 (Apple Silicon), you need to use **conda** and install protobuf by running the following command:
+
+```bash
+conda install "protobuf>=3.20.3, <4" -y
+````
+
 2. **Advanced** <br> 
    - If you expect to connect to, or work with, cloud providers (Azure/Google Cloud/S3), you can install additional packages. This is not 
    part of the regular requirements since not all users work with those platforms. Using this option reduces the dependencies and the size 

docs/model-monitoring/dashboards/model-monitoring-details.json

@@ -480,174 +480,6 @@
       "transparent": true,
       "type": "table"
     },
-    {
-      "datasource": "model-monitoring",
-      "description": "Feature analysis of the latest batch",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "align": "center",
-            "displayMode": "auto",
-            "filterable": false,
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Expected Min"
-            },
-            "properties": [
-              {
-                "id": "noValue",
-                "value": "N/A"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Expected Mean"
-            },
-            "properties": [
-              {
-                "id": "noValue",
-                "value": "N/A"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Expected Max"
-            },
-            "properties": [
-              {
-                "id": "noValue",
-                "value": "N/A"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "tvd"
-            },
-            "properties": [
-              {
-                "id": "displayName",
-                "value": "TVD"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "hellinger"
-            },
-            "properties": [
-              {
-                "id": "displayName",
-                "value": "Hellinger"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "kld"
-            },
-            "properties": [
-              {
-                "id": "displayName",
-                "value": "KLD"
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 6
-      },
-      "id": 29,
-      "options": {
-        "footer": {
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": false,
-            "displayName": "Feature"
-          }
-        ]
-      },
-      "pluginVersion": "9.2.15",
-      "targets": [
-        {
-          "datasource": "model-monitoring",
-          "rawQuery": true,
-          "refId": "A",
-          "target": "target_endpoint=individual_feature_analysis;endpoint_id=$MODELENDPOINT;project=$PROJECT",
-          "type": "table"
-        }
-      ],
-      "title": "Features Analysis",
-      "transformations": [
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "count": true,
-              "idx": true,
-              "model": true
-            },
-            "indexByName": {
-              "actual_max": 3,
-              "actual_mean": 2,
-              "actual_min": 1,
-              "expected_max": 4,
-              "expected_mean": 5,
-              "expected_min": 6,
-              "feature_name": 0
-            },
-            "renameByName": {
-              "actual_max": "Actual Max",
-              "actual_mean": "Actual Mean",
-              "actual_min": "Actual Min",
-              "expected_max": "Expected Min",
-              "expected_mean": "Expected Mean",
-              "expected_min": "Expected Max",
-              "feature_name": "Feature"
-            }
-          }
-        }
-      ],
-      "transparent": true,
-      "type": "table"
-    },
     {
       "aliasColors": {},
       "bars": false,

mlrun/common/schemas/notification.py

@@ -22,11 +22,38 @@
 
 
 class NotificationKind(mlrun.common.types.StrEnum):
-    console = "console"
-    git = "git"
-    ipython = "ipython"
-    slack = "slack"
-    webhook = "webhook"
+    """Currently, the supported notification kinds and their params are as follows:"""
+
+    console: str = "console"
+    """no params, local only"""
+
+    git: str = "git"
+    """
+    **token** - The git token to use for the git notification.\n
+    **repo** - The git repo to which to send the notification.\n
+    **issue** - The git issue to which to send the notification.\n
+    **merge_request** -
+     In GitLab (as opposed to GitHub), merge requests and issues are separate entities.
+     If using merge request, the issue will be ignored, and vice versa.\n
+    **server** - The git server to which to send the notification.\n
+    **gitlab** - (bool) Whether the git server is GitLab or not.\n
+    """
+
+    ipython: str = "ipython"
+    """no params, local only"""
+
+    slack: str = "slack"
+    """**webhook** - The slack webhook to which to send the notification."""
+
+    webhook: str = "webhook"
+    """
+    **url** - The webhook url to which to send the notification.\n
+    **method** - The http method to use when sending the notification (GET, POST, PUT, etc…).\n
+    **headers** - (dict) The http headers to send with the notification.\n
+    **override_body** - (dict) The body to send with the notification.\n
+    **verify_ssl** -
+     (bool) Whether SSL certificates are validated during HTTP requests or not, The default is set to True.
+    """
 
 
 class NotificationSeverity(mlrun.common.types.StrEnum):

mlrun/execution.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 import uuid
 from copy import deepcopy
@@ -168,6 +169,8 @@ def log_level(self):
     @log_level.setter
     def log_level(self, value: str):
         """Set the logging level, e.g. 'debug', 'info', 'error'"""
+        level = logging.getLevelName(value.upper())
+        self._logger.set_logger_level(level)
         self._log_level = value
 
     @property

mlrun/k8s_utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import re
+import warnings
 
 import kubernetes.client
 
@@ -133,7 +134,7 @@ def sanitize_label_value(value: str) -> str:
     return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
 
 
-def verify_label_key(key: str):
+def verify_label_key(key: str, allow_k8s_prefix: bool = False):
     """
     Verify that the label key is valid for Kubernetes.
     Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
@@ -146,6 +147,10 @@ def verify_label_key(key: str):
         name = parts[0]
     elif len(parts) == 2:
         prefix, name = parts
+        if len(name) == 0:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Label key name cannot be empty when a prefix is set"
+            )
         if len(prefix) == 0:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Label key prefix cannot be empty"
@@ -173,7 +178,13 @@ def verify_label_key(key: str):
         mlrun.utils.regex.qualified_name,
     )
 
-    if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
+    # Allow the use of Kubernetes reserved prefixes ('k8s.io/' or 'kubernetes.io/')
+    # only when setting node selectors, not when adding new labels.
+    if (
+        key.startswith("k8s.io/")
+        or key.startswith("kubernetes.io/")
+        and not allow_k8s_prefix
+    ):
         raise mlrun.errors.MLRunInvalidArgumentError(
             "Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
         )
@@ -185,3 +196,38 @@ def verify_label_value(value, label_key):
         value,
         mlrun.utils.regex.label_value,
     )
+
+
+def validate_node_selectors(
+    node_selectors: dict[str, str], raise_on_error: bool = True
+) -> bool:
+    """
+    Ensures that user-defined node selectors adhere to Kubernetes label standards:
+    - Validates that each key conforms to Kubernetes naming conventions, with specific rules for name and prefix.
+    - Ensures values comply with Kubernetes label value rules.
+    - If raise_on_error is True, raises errors for invalid selectors.
+    - If raise_on_error is False, logs warnings for invalid selectors.
+    """
+
+    # Helper function for handling errors or warnings
+    def handle_invalid(message):
+        if raise_on_error:
+            raise
+        else:
+            warnings.warn(
+                f"{message}\n"
+                f"The node selector you’ve set does not meet the validation rules for the current Kubernetes version. "
+                f"Please note that invalid node selectors may cause issues with function scheduling."
+            )
+
+    node_selectors = node_selectors or {}
+    for key, value in node_selectors.items():
+        try:
+            verify_label_key(key, allow_k8s_prefix=True)
+            verify_label_value(value, label_key=key)
+        except mlrun.errors.MLRunInvalidArgumentError as err:
+            # An error or warning is raised by handle_invalid due to validation failure.
+            # Returning False indicates validation failed, allowing us to exit the function.
+            handle_invalid(str(err))
+            return False
+    return True

mlrun/model.py

@@ -681,7 +681,8 @@ def _resolve_requirements(requirements: list, requirements_file: str = "") -> li
 class Notification(ModelObj):
     """Notification object
 
-    :param kind: notification implementation kind - slack, webhook, etc.
+    :param kind: notification implementation kind - slack, webhook, etc. See
+        :py:class:`mlrun.common.schemas.notification.NotificationKind`
     :param name: for logging and identification
     :param message: message content in the notification
     :param severity: severity to display in the notification

mlrun/projects/project.py

@@ -26,6 +26,7 @@
 import uuid
 import warnings
 import zipfile
+from copy import deepcopy
 from os import environ, makedirs, path
 from typing import Callable, Optional, Union
 
@@ -252,8 +253,7 @@ def setup(project):
         project.spec.description = description
 
     if default_function_node_selector:
-        for key, val in default_function_node_selector.items():
-            project.spec.default_function_node_selector[key] = val
+        project.spec.default_function_node_selector = default_function_node_selector
 
     if parameters:
         # Enable setting project parameters at load time, can be used to customize the project_setup
@@ -875,7 +875,7 @@ def __init__(
         # in a tuple where the first index is the packager module's path (str) and the second is a flag (bool) for
         # whether it is mandatory for a run (raise exception on collection error) or not.
         self.custom_packagers = custom_packagers or []
-        self.default_function_node_selector = default_function_node_selector or {}
+        self._default_function_node_selector = default_function_node_selector or {}
 
     @property
     def source(self) -> str:
@@ -1050,6 +1050,14 @@ def remove_artifact(self, key):
         if key in self._artifacts:
             del self._artifacts[key]
 
+    @property
+    def default_function_node_selector(self):
+        return self._default_function_node_selector
+
+    @default_function_node_selector.setter
+    def default_function_node_selector(self, node_selector: dict[str, str]):
+        self._default_function_node_selector = deepcopy(node_selector)
+
     @property
     def build(self) -> ImageBuilder:
         return self._build