Merge remote-tracking branch 'origin/master' into modelUploadStorageR…

…equestSize
Clarifai · Jan 14, 2025 · e8a3639 · e8a3639
2 parents 0942371 + 2a12775
commit e8a3639
Show file tree

Hide file tree

Showing 7 changed files with 197 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,19 @@
+## [[11.0.1]](https://github.com/Clarifai/clarifai-python/releases/tag/11.0.1) - [PyPI](https://pypi.org/project/clarifai/11.0.1/) - 2025-01-14
+
+### Changed
+
+ - Take user_id from Env variable [(#477)] (https://github.com/Clarifai/clarifai-python/pull/477)
+ - Added HF token Validation [(#476)] (https://github.com/Clarifai/clarifai-python/pull/476)
+ - Fix Model prediction methods when configured with a dedicated compute_cluster_id and nodepool_id [(#475)] (https://github.com/Clarifai/clarifai-python/pull/475)
+ - Fix model upload issues [(#474)] (https://github.com/Clarifai/clarifai-python/pull/474)
+ - Improved error logging [(#473)] (https://github.com/Clarifai/clarifai-python/pull/473)
+
+## [[11.0.0]](https://github.com/Clarifai/clarifai-python/releases/tag/11.0.0) - [PyPI](https://pypi.org/project/clarifai/11.0.0/) - 2025-01-07
+
+### Changed
+ - Changed labels to optional in Dataloaders to support Data Ingestion pipelines in clarifai-datautils library [(#471)] (https://github.com/Clarifai/clarifai-python/pull/471)
+
+
 ## [[10.11.1]](https://github.com/Clarifai/clarifai-python/releases/tag/10.11.1) - [PyPI](https://pypi.org/project/clarifai/10.11.1/) - 2024-12-20
 
 ### Added

diff --git a/clarifai/__init__.py b/clarifai/__init__.py
@@ -1 +1 @@
-__version__ = "10.11.1"
+__version__ = "11.0.1"
diff --git a/clarifai/client/dataset.py b/clarifai/client/dataset.py
@@ -281,9 +281,9 @@ def _delete_failed_inputs(self,
       failed_input_ids = list(set(failed_input_ids) - set(duplicate_input_ids))
       duplicate_details = [[
           input_ids[id], id, "Input has a duplicate ID.",
-          dataset_obj.data_generator[input_ids[id]].image_path,
-          dataset_obj.data_generator[input_ids[id]].labels,
-          dataset_obj.data_generator[input_ids[id]].metadata
+          getattr(dataset_obj.data_generator[input_ids[id]], 'image_path', None),
+          getattr(dataset_obj.data_generator[input_ids[id]], 'labels', None),
+          getattr(dataset_obj.data_generator[input_ids[id]], 'metadata', None)
       ] for id in duplicate_input_ids]
       duplicate_table = tabulate(
           duplicate_details,
@@ -386,7 +386,7 @@ def _retry_uploads(self, failed_input_ids: List[int],
           tablefmt="grid")
       timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
       self.logger.warning(
-          f"{timestamp}\nFailed to upload {len(failed_retrying_inputs)} inputs in current batch {batch_no}:\n{failed_table}\n\n"
+          f"{timestamp}\nFailed to upload {len(failed_retrying_inputs)} inputs in current batch {batch_no} due to {retry_response}:\n{failed_table}\n\n"
       )
 
   def _data_upload(self,

diff --git a/clarifai/client/model.py b/clarifai/client/model.py
@@ -503,6 +503,7 @@ def predict_by_filepath(self,
                           compute_cluster_id: str = None,
                           nodepool_id: str = None,
                           deployment_id: str = None,
+                          user_id: str = None,
                           inference_params: Dict = {},
                           output_config: Dict = {}):
     """Predicts the model based on the given filepath.
@@ -534,14 +535,15 @@ def predict_by_filepath(self,
       file_bytes = f.read()
 
     return self.predict_by_bytes(file_bytes, input_type, compute_cluster_id, nodepool_id,
-                                 deployment_id, inference_params, output_config)
+                                 deployment_id, user_id, inference_params, output_config)
 
   def predict_by_bytes(self,
                        input_bytes: bytes,
                        input_type: str = None,
                        compute_cluster_id: str = None,
                        nodepool_id: str = None,
                        deployment_id: str = None,
+                       user_id: str = None,
                        inference_params: Dict = {},
                        output_config: Dict = {}):
     """Predicts the model based on the given bytes.
@@ -581,11 +583,23 @@ def predict_by_bytes(self,
 
     runner_selector = None
     if deployment_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with deployment ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Deployment.get_runner_selector(
-          user_id=self.user_id, deployment_id=deployment_id)
+          user_id=user_id, deployment_id=deployment_id)
     elif compute_cluster_id and nodepool_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with compute cluster ID and nodepool ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Nodepool.get_runner_selector(
-          user_id=self.user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
+          user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
 
     return self.predict(
         inputs=[input_proto],
@@ -599,6 +613,7 @@ def predict_by_url(self,
                      compute_cluster_id: str = None,
                      nodepool_id: str = None,
                      deployment_id: str = None,
+                     user_id: str = None,
                      inference_params: Dict = {},
                      output_config: Dict = {}):
     """Predicts the model based on the given URL.
@@ -639,11 +654,23 @@ def predict_by_url(self,
 
     runner_selector = None
     if deployment_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with deployment ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Deployment.get_runner_selector(
-          user_id=self.user_id, deployment_id=deployment_id)
+          user_id=user_id, deployment_id=deployment_id)
     elif compute_cluster_id and nodepool_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with compute cluster ID and nodepool ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Nodepool.get_runner_selector(
-          user_id=self.user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
+          user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
 
     return self.predict(
         inputs=[input_proto],
@@ -712,6 +739,7 @@ def generate_by_filepath(self,
                            compute_cluster_id: str = None,
                            nodepool_id: str = None,
                            deployment_id: str = None,
+                           user_id: str = None,
                            inference_params: Dict = {},
                            output_config: Dict = {}):
     """Generate the stream output on model based on the given filepath.
@@ -748,6 +776,7 @@ def generate_by_filepath(self,
         compute_cluster_id=compute_cluster_id,
         nodepool_id=nodepool_id,
         deployment_id=deployment_id,
+        user_id=user_id,
         inference_params=inference_params,
         output_config=output_config)
 
@@ -757,6 +786,7 @@ def generate_by_bytes(self,
                         compute_cluster_id: str = None,
                         nodepool_id: str = None,
                         deployment_id: str = None,
+                        user_id: str = None,
                         inference_params: Dict = {},
                         output_config: Dict = {}):
     """Generate the stream output on model based on the given bytes.
@@ -798,11 +828,21 @@ def generate_by_bytes(self,
 
     runner_selector = None
     if deployment_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with deployment ID, please provide user_id in the method call."
+        )
       runner_selector = Deployment.get_runner_selector(
-          user_id=self.user_id, deployment_id=deployment_id)
+          user_id=user_id, deployment_id=deployment_id)
     elif compute_cluster_id and nodepool_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with compute cluster ID and nodepool ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Nodepool.get_runner_selector(
-          user_id=self.user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
+          user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
 
     return self.generate(
         inputs=[input_proto],
@@ -816,6 +856,7 @@ def generate_by_url(self,
                       compute_cluster_id: str = None,
                       nodepool_id: str = None,
                       deployment_id: str = None,
+                      user_id: str = None,
                       inference_params: Dict = {},
                       output_config: Dict = {}):
     """Generate the stream output on model based on the given URL.
@@ -857,11 +898,23 @@ def generate_by_url(self,
 
     runner_selector = None
     if deployment_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with deployment ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Deployment.get_runner_selector(
-          user_id=self.user_id, deployment_id=deployment_id)
+          user_id=user_id, deployment_id=deployment_id)
     elif compute_cluster_id and nodepool_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with compute cluster ID and nodepool ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Nodepool.get_runner_selector(
-          user_id=self.user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
+          user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
 
     return self.generate(
         inputs=[input_proto],
@@ -930,6 +983,7 @@ def stream_by_filepath(self,
                          compute_cluster_id: str = None,
                          nodepool_id: str = None,
                          deployment_id: str = None,
+                         user_id: str = None,
                          inference_params: Dict = {},
                          output_config: Dict = {}):
     """Stream the model output based on the given filepath.
@@ -964,6 +1018,7 @@ def stream_by_filepath(self,
         compute_cluster_id=compute_cluster_id,
         nodepool_id=nodepool_id,
         deployment_id=deployment_id,
+        user_id=user_id,
         inference_params=inference_params,
         output_config=output_config)
 
@@ -973,6 +1028,7 @@ def stream_by_bytes(self,
                       compute_cluster_id: str = None,
                       nodepool_id: str = None,
                       deployment_id: str = None,
+                      user_id: str = None,
                       inference_params: Dict = {},
                       output_config: Dict = {}):
     """Stream the model output based on the given bytes.
@@ -1016,11 +1072,23 @@ def input_generator():
 
     runner_selector = None
     if deployment_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with deployment ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Deployment.get_runner_selector(
-          user_id=self.user_id, deployment_id=deployment_id)
+          user_id=user_id, deployment_id=deployment_id)
     elif compute_cluster_id and nodepool_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with compute cluster ID and nodepool ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Nodepool.get_runner_selector(
-          user_id=self.user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
+          user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
 
     return self.stream(
         inputs=input_generator(),
@@ -1034,6 +1102,7 @@ def stream_by_url(self,
                     compute_cluster_id: str = None,
                     nodepool_id: str = None,
                     deployment_id: str = None,
+                    user_id: str = None,
                     inference_params: Dict = {},
                     output_config: Dict = {}):
     """Stream the model output based on the given URL.
@@ -1075,11 +1144,23 @@ def input_generator():
 
     runner_selector = None
     if deployment_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with deployment ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Deployment.get_runner_selector(
-          user_id=self.user_id, deployment_id=deployment_id)
+          user_id=user_id, deployment_id=deployment_id)
     elif compute_cluster_id and nodepool_id:
+      if not user_id and not os.environ.get('CLARIFAI_USER_ID'):
+        raise UserError(
+            "User ID is required for model prediction with compute cluster ID and nodepool ID, please provide user_id in the method call."
+        )
+      if not user_id:
+        user_id = os.environ.get('CLARIFAI_USER_ID')
       runner_selector = Nodepool.get_runner_selector(
-          user_id=self.user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
+          user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id)
 
     return self.stream(
         inputs=input_generator(),