diff --git a/docs/my-website/docs/proxy/guardrails/quick_start.md b/docs/my-website/docs/proxy/guardrails/quick_start.md index 29f8a7b551cd..a229e9eb57ad 100644 --- a/docs/my-website/docs/proxy/guardrails/quick_start.md +++ b/docs/my-website/docs/proxy/guardrails/quick_start.md @@ -112,14 +112,49 @@ curl -i http://localhost:4000/v1/chat/completions \ +## **Using Guardrails client side** + + +### ✨ View available guardrails (/guardrails/list) + +Show available guardrails on the proxy server. This makes it easier for developers to know what guardrails are available / can be used. + +```shell +curl -X GET 'http://0.0.0.0:4000/guardrails/list' +``` + +Expected response + +```json +{ + "guardrails": [ + { + "guardrail_name": "bedrock-pre-guard", + "guardrail_info": { + "params": [ + { + "name": "toxicity_score", + "type": "float", + "description": "Score between 0-1 indicating content toxicity level" + }, + { + "name": "pii_detection", + "type": "boolean" + } + ] + } + } + ] +} +``` + -## Advanced ### ✨ Pass additional parameters to guardrail :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/#trial) ::: @@ -196,11 +231,40 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +## **Proxy Admin Controls** + +### ✨ Monitoring Guardrails + +Monitor which guardrails were executed and whether they passed or failed. e.g. guardrail going rogue and failing requests we don't intend to fail + +:::info + +✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/#trial) + +::: + +### Setup + +1. Connect LiteLLM to a [supported logging provider](../logging) +2. Make a request with a `guardrails` parameter +3. Check your logging provider for the guardrail trace + +#### Traced Guardrail Success + + + +#### Traced Guardrail Failure + + + + + + ### ✨ Control Guardrails per Project (API Key) :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/#trial) ::: @@ -262,7 +326,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/#trial) ::: @@ -320,22 +384,6 @@ The `pii_masking` guardrail ran on this request because api key=sk-jNm1Zar7XfNdZ -### ✨ List guardrails - -Show available guardrails on the proxy server. This makes it easier for developers to know what guardrails are available / can be used. - -```shell -curl -X GET 'http://0.0.0.0:4000/guardrails/list' -``` - -Expected response - -```json -{ - "guardrails": ["aporia-pre-guard", "aporia-post-guard"] -} -``` - ## Spec: `guardrails` Parameter The `guardrails` parameter can be passed to any LiteLLM Proxy endpoint (`/chat/completions`, `/completions`, `/embeddings`). diff --git a/docs/my-website/img/gd_fail.png b/docs/my-website/img/gd_fail.png new file mode 100644 index 000000000000..2766e57a80b1 Binary files /dev/null and b/docs/my-website/img/gd_fail.png differ diff --git a/docs/my-website/img/gd_success.png b/docs/my-website/img/gd_success.png new file mode 100644 index 000000000000..979e3dfc5760 Binary files /dev/null and b/docs/my-website/img/gd_success.png differ diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 7706d9ab6381..2aac83327a1d 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1,8 +1,9 @@ -from typing import Dict, List, Optional, Union +from typing import Dict, List, Literal, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.types.guardrails import DynamicGuardrailParams, GuardrailEventHooks +from litellm.types.utils import StandardLoggingGuardrailInformation class CustomGuardrail(CustomLogger): @@ -119,3 +120,101 @@ def _validate_premium_user(self) -> bool: ) return False return True + + def add_standard_logging_guardrail_information_to_request_data( + self, + guardrail_json_response: Union[Exception, str, dict], + request_data: dict, + guardrail_status: Literal["success", "failure"], + ) -> None: + """ + Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc. + """ + from litellm.proxy.proxy_server import premium_user + + if premium_user is not True: + verbose_logger.warning( + f"Guardrail Tracing is only available for premium users. Skipping guardrail logging for guardrail={self.guardrail_name} event_hook={self.event_hook}" + ) + return + if isinstance(guardrail_json_response, Exception): + guardrail_json_response = str(guardrail_json_response) + slg = StandardLoggingGuardrailInformation( + guardrail_name=self.guardrail_name, + guardrail_mode=self.event_hook, + guardrail_response=guardrail_json_response, + guardrail_status=guardrail_status, + ) + if "metadata" in request_data: + request_data["metadata"]["standard_logging_guardrail_information"] = slg + elif "litellm_metadata" in request_data: + request_data["litellm_metadata"][ + "standard_logging_guardrail_information" + ] = slg + else: + verbose_logger.warning( + "unable to log guardrail information. No metadata found in request_data" + ) + + +def log_guardrail_information(func): + """ + Decorator to add standard logging guardrail information to any function + + Add this decorator to ensure your guardrail response is logged to DataDog, OTEL, s3, GCS etc. + + Logs for: + - pre_call + - during_call + - TODO: log post_call. This is more involved since the logs are sent to DD, s3 before the guardrail is even run + """ + import asyncio + import functools + + def process_response(self, response, request_data): + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=response, + request_data=request_data, + guardrail_status="success", + ) + return response + + def process_error(self, e, request_data): + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=e, + request_data=request_data, + guardrail_status="failure", + ) + raise e + + @functools.wraps(func) + async def async_wrapper(*args, **kwargs): + self: CustomGuardrail = args[0] + request_data: Optional[dict] = ( + kwargs.get("data") or kwargs.get("request_data") or {} + ) + try: + response = await func(*args, **kwargs) + return process_response(self, response, request_data) + except Exception as e: + return process_error(self, e, request_data) + + @functools.wraps(func) + def sync_wrapper(*args, **kwargs): + self: CustomGuardrail = args[0] + request_data: Optional[dict] = ( + kwargs.get("data") or kwargs.get("request_data") or {} + ) + try: + response = func(*args, **kwargs) + return process_response(self, response, request_data) + except Exception as e: + return process_error(self, e, request_data) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if asyncio.iscoroutinefunction(func): + return async_wrapper(*args, **kwargs) + return sync_wrapper(*args, **kwargs) + + return wrapper diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index abdd327ff02f..899af525c80d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3038,6 +3038,9 @@ def get_standard_logging_object_payload( response_cost_failure_debug_info=kwargs.get( "response_cost_failure_debug_information" ), + guardrail_information=metadata.get( + "standard_logging_guardrail_information", None + ), ) return payload diff --git a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai.py b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai.py index 9e3fdde3be18..4e37b4eb849c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai.py @@ -19,7 +19,10 @@ import litellm from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.litellm_core_utils.logging_utils import ( convert_litellm_response_object_to_str, ) @@ -142,6 +145,7 @@ async def make_aporia_api_request( }, ) + @log_guardrail_information async def async_post_call_success_hook( self, data: dict, @@ -173,6 +177,7 @@ async def async_post_call_success_hook( pass + @log_guardrail_information async def async_moderation_hook( ### 👈 KEY CHANGE ### self, data: dict, diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 1b6880581c03..828e15c2775d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -19,7 +19,10 @@ import litellm from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -231,6 +234,7 @@ async def make_bedrock_api_request( response.text, ) + @log_guardrail_information async def async_moderation_hook( ### 👈 KEY CHANGE ### self, data: dict, @@ -263,6 +267,7 @@ async def async_moderation_hook( ### 👈 KEY CHANGE ### ) pass + @log_guardrail_information async def async_post_call_success_hook( self, data: dict, diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_guardrail.py index 4e6bab635242..a45343b37dbe 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_guardrail.py @@ -3,7 +3,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.proxy._types import UserAPIKeyAuth @@ -17,6 +20,7 @@ def __init__( super().__init__(**kwargs) + @log_guardrail_information async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -55,6 +59,7 @@ async def async_pre_call_hook( return data + @log_guardrail_information async def async_moderation_hook( self, data: dict, @@ -84,6 +89,7 @@ async def async_moderation_hook( if "litellm" in _content.lower(): raise ValueError("Guardrail failed words - `litellm` detected") + @log_guardrail_information async def async_post_call_success_hook( self, data: dict, diff --git a/litellm/proxy/guardrails/guardrail_hooks/guardrails_ai.py b/litellm/proxy/guardrails/guardrail_hooks/guardrails_ai.py index 1500e8c25aa1..1a2c5a217b0f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/guardrails_ai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/guardrails_ai.py @@ -12,7 +12,10 @@ import litellm from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) @@ -79,6 +82,7 @@ async def make_guardrails_ai_api_request(self, llm_output: str, request_data: di ) return _json_response + @log_guardrail_information async def async_post_call_success_hook( self, data: dict, diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai.py index 6f05d366fa47..f55b78b0a929 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai.py @@ -20,7 +20,10 @@ import litellm from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -294,6 +297,7 @@ async def _check( # noqa: PLR0915 """ self._check_response_flagged(response=response.json()) + @log_guardrail_information async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -330,6 +334,7 @@ async def async_pre_call_hook( data=data, user_api_key_dict=user_api_key_dict, call_type=call_type ) + @log_guardrail_information async def async_moderation_hook( ### 👈 KEY CHANGE ### self, data: dict, diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index fb58170cf5b2..86d2c8b25add 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -20,7 +20,10 @@ from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.utils import ( @@ -205,6 +208,7 @@ async def check_pii( except Exception as e: raise e + @log_guardrail_information async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -257,6 +261,7 @@ async def async_pre_call_hook( except Exception as e: raise e + @log_guardrail_information def logging_hook( self, kwargs: dict, result: Any, call_type: str ) -> Tuple[dict, Any]: @@ -289,6 +294,7 @@ def run_in_new_loop(): # No running event loop, we can safely run in this thread return run_in_new_loop() + @log_guardrail_information async def async_logging_hook( self, kwargs: dict, result: Any, call_type: str ) -> Tuple[dict, Any]: @@ -333,6 +339,7 @@ async def async_logging_hook( return kwargs, result + @log_guardrail_information async def async_post_call_success_hook( # type: ignore self, data: dict, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 376aaa2bb163..dd151e36e463 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -11,11 +11,13 @@ model_list: litellm_params: model: bedrock/* - guardrails: - guardrail_name: "bedrock-pre-guard" litellm_params: guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" + mode: "during_call" + guardrailIdentifier: ff6ujrregl1q # your guardrail ID on bedrock + guardrailVersion: "DRAFT" # your guardrail version on bedrock mode: "post_call" guardrailIdentifier: ff6ujrregl1q guardrailVersion: "DRAFT" diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 3baf63f3b21d..86e87fbe3e4e 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -105,9 +105,10 @@ class LitellmParams(TypedDict): guard_name: Optional[str] -class Guardrail(TypedDict): +class Guardrail(TypedDict, total=False): guardrail_name: str litellm_params: LitellmParams + guardrail_info: Optional[Dict] class guardrailConfig(TypedDict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 957ce3ff5b91..2eb4ca8bbed1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -21,6 +21,7 @@ from typing_extensions import Callable, Dict, Required, TypedDict, override from ..litellm_core_utils.core_helpers import map_finish_reason +from .guardrails import GuardrailEventHooks from .llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionUsageBlock, @@ -1500,6 +1501,13 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): llm_provider: Optional[str] +class StandardLoggingGuardrailInformation(TypedDict, total=False): + guardrail_name: Optional[str] + guardrail_mode: Optional[GuardrailEventHooks] + guardrail_response: Optional[Union[dict, str]] + guardrail_status: Literal["success", "failure"] + + StandardLoggingPayloadStatus = Literal["success", "failure"] @@ -1539,6 +1547,7 @@ class StandardLoggingPayload(TypedDict): error_information: Optional[StandardLoggingPayloadErrorInformation] model_parameters: dict hidden_params: StandardLoggingHiddenParams + guardrail_information: Optional[StandardLoggingGuardrailInformation] from typing import AsyncIterator, Iterator