update docs

SeldonIO · Dec 15, 2023 · 36e4f9d · 36e4f9d
1 parent 13c274b
commit 36e4f9d
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 0 deletions.
diff --git a/runtimes/huggingface/README.md b/runtimes/huggingface/README.md
@@ -66,6 +66,25 @@ Models in the HuggingFace hub can be loaded by specifying their name in `paramet
 If `parameters.extra.pretrained_model` is specified, it takes precedence over `parameters.uri`.
 ````
 
+#### Model Inference
+Model inference is done by HuggingFace pipeline. It allows users to run inference on a batch of inputs. Extra inference kwargs can be kept in `parameters.extra`.
+```{code-block} json
+
+{
+    "inputs": [
+        {
+            "name": "text_inputs",
+            "shape": [1],
+            "datatype": "BYTES",
+            "data": ["My kitten's name is JoJo,","Tell me a story:"],
+        }
+    ],
+    "parameters": {
+        "extra":{"max_new_tokens": 200,"return_full_text": false}
+    }
+}
+```
+
 ### Reference
 
 You can find the full reference of the accepted extra settings for the

diff --git a/runtimes/huggingface/mlserver_huggingface/codecs/base.py b/runtimes/huggingface/mlserver_huggingface/codecs/base.py
@@ -171,6 +171,10 @@ def encode_request(cls, payload: Dict[str, Any], **kwargs) -> InferenceRequest:
 
     @classmethod
     def decode_request(cls, request: InferenceRequest) -> Dict[str, Any]:
+        """
+        Decode Inference requst into dictionary
+        extra Inference kwargs can be kept in 'InferenceRequest.parameters.extra'
+        """
         values = {}
         field_codecs = cls._find_decode_codecs(request)
         for item in request.inputs: