phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

build(ui): enable adding openmeter via proxy ui

Browse source
This commit is contained in:
Krrish Dholakia 2024-05-01 21:16:23 -07:00
parent dbf999f56e
commit cdd3e1eef3
5 changed files with 214 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/my-website/docs/proxy/logging.md
View file

@ -600,7 +600,7 @@ Bill customers according to their LLM API usage with [OpenMeter](../observabilit
```bash
# from https://openmeter.cloud
export OPENMETER_API_ENDPOINT=""
export OPENMETER_API_ENDPOINT="" # defaults to https://openmeter.cloud
export OPENMETER_API_KEY=""
```

11
litellm/integrations/openmeter.py
View file

@ -38,9 +38,6 @@ class OpenMeterLogger(CustomLogger):
in the environment
"""
missing_keys = []
if litellm.get_secret("OPENMETER_API_ENDPOINT", None) is None:
missing_keys.append("OPENMETER_API_ENDPOINT")
if litellm.get_secret("OPENMETER_API_KEY", None) is None:
missing_keys.append("OPENMETER_API_KEY")
@ -74,7 +71,9 @@ class OpenMeterLogger(CustomLogger):
}
def log_success_event(self, kwargs, response_obj, start_time, end_time):
_url = litellm.get_secret("OPENMETER_API_ENDPOINT")
_url = litellm.get_secret(
"OPENMETER_API_ENDPOINT", default_value="https://openmeter.cloud"
)
if _url.endswith("/"):
_url += "api/v1/events"
else:
@ -93,7 +92,9 @@ class OpenMeterLogger(CustomLogger):
)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
_url = litellm.get_secret("OPENMETER_API_ENDPOINT")
_url = litellm.get_secret(
"OPENMETER_API_ENDPOINT", default_value="https://openmeter.cloud"
)
if _url.endswith("/"):
_url += "api/v1/events"
else:

161
litellm/proxy/proxy_server.py
View file

@ -2693,9 +2693,10 @@ class ProxyConfig:
environment_variables = config_data.get("environment_variables", {})
for k, v in environment_variables.items():
try:
decoded_b64 = base64.b64decode(v)
value = decrypt_value(value=decoded_b64, master_key=master_key) # type: ignore
os.environ[k] = value
if v is not None:
decoded_b64 = base64.b64decode(v)
value = decrypt_value(value=decoded_b64, master_key=master_key) # type: ignore
os.environ[k] = value
except Exception as e:
verbose_proxy_logger.error(
"Error setting env variable: %s - %s", k, str(e)
@ -8644,6 +8645,13 @@ async def update_config(config_info: ConfigYAML):
_existing_settings = config["general_settings"]
for k, v in updated_general_settings.items():
# overwrite existing settings with updated values
if k == "alert_to_webhook_url":
# check if slack is already enabled. if not, enable it
if "slack" not in _existing_settings:
if "alerting" not in _existing_settings:
_existing_settings["alerting"] = ["slack"]
elif isinstance(_existing_settings["alerting"], list):
_existing_settings["alerting"].append("slack")
_existing_settings[k] = v
config["general_settings"] = _existing_settings
@ -8758,7 +8766,25 @@ async def get_config():
"""
for _callback in _success_callbacks:
if _callback == "langfuse":
if _callback == "openmeter":
env_vars = [
"OPENMETER_API_KEY",
]
env_vars_dict = {}
for _var in env_vars:
env_variable = environment_variables.get(_var, None)
if env_variable is None:
env_vars_dict[_var] = None
else:
# decode + decrypt the value
decoded_b64 = base64.b64decode(env_variable)
_decrypted_value = decrypt_value(
value=decoded_b64, master_key=master_key
)
env_vars_dict[_var] = _decrypted_value
_data_to_return.append({"name": _callback, "variables": env_vars_dict})
elif _callback == "langfuse":
_langfuse_vars = [
"LANGFUSE_PUBLIC_KEY",
"LANGFUSE_SECRET_KEY",
@ -8898,9 +8924,9 @@ async def test_endpoint(request: Request):
)
async def health_services_endpoint(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
service: Literal["slack_budget_alerts", "langfuse", "slack"] = fastapi.Query(
description="Specify the service being hit."
),
service: Literal[
"slack_budget_alerts", "langfuse", "slack", "openmeter"
] = fastapi.Query(description="Specify the service being hit."),
):
"""
Hidden endpoint.
@ -8914,7 +8940,7 @@ async def health_services_endpoint(
raise HTTPException(
status_code=400, detail={"error": "Service must be specified."}
)
if service not in ["slack_budget_alerts", "langfuse", "slack"]:
if service not in ["slack_budget_alerts", "langfuse", "slack", "openmeter"]:
raise HTTPException(
status_code=400,
detail={
@ -8922,6 +8948,18 @@ async def health_services_endpoint(
},
)
if service == "openmeter":
_ = await litellm.acompletion(
model="openai/litellm-mock-response-model",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
user="litellm:/health/services",
mock_response="This is a mock response",
)
return {
"status": "success",
"message": "Mock LLM request made - check openmeter.",
}
if service == "langfuse":
from litellm.integrations.langfuse import LangFuseLogger
@ -8938,63 +8976,66 @@ async def health_services_endpoint(
"message": "Mock LLM request made - check langfuse.",
}
if "slack" in general_settings.get("alerting", []):
# test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
# check if user has opted into unique_alert_webhooks
if (
proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
is not None
):
for (
alert_type
) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url:
"""
"llm_exceptions",
"llm_too_slow",
"llm_requests_hanging",
"budget_alerts",
"db_exceptions",
"""
# only test alert if it's in active alert types
if (
proxy_logging_obj.slack_alerting_instance.alert_types
is not None
and alert_type
not in proxy_logging_obj.slack_alerting_instance.alert_types
):
continue
test_message = "default test message"
if alert_type == "llm_exceptions":
test_message = f"LLM Exception test alert"
elif alert_type == "llm_too_slow":
test_message = f"LLM Too Slow test alert"
elif alert_type == "llm_requests_hanging":
test_message = f"LLM Requests Hanging test alert"
elif alert_type == "budget_alerts":
test_message = f"Budget Alert test alert"
elif alert_type == "db_exceptions":
test_message = f"DB Exception test alert"
if service == "slack" or service == "slack_budget_alerts":
if "slack" in general_settings.get("alerting", []):
# test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
# check if user has opted into unique_alert_webhooks
if (
proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
is not None
):
for (
alert_type
) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url:
"""
"llm_exceptions",
"llm_too_slow",
"llm_requests_hanging",
"budget_alerts",
"db_exceptions",
"""
# only test alert if it's in active alert types
if (
proxy_logging_obj.slack_alerting_instance.alert_types
is not None
and alert_type
not in proxy_logging_obj.slack_alerting_instance.alert_types
):
continue
test_message = "default test message"
if alert_type == "llm_exceptions":
test_message = f"LLM Exception test alert"
elif alert_type == "llm_too_slow":
test_message = f"LLM Too Slow test alert"
elif alert_type == "llm_requests_hanging":
test_message = f"LLM Requests Hanging test alert"
elif alert_type == "budget_alerts":
test_message = f"Budget Alert test alert"
elif alert_type == "db_exceptions":
test_message = f"DB Exception test alert"
await proxy_logging_obj.alerting_handler(
message=test_message, level="Low", alert_type=alert_type
)
else:
await proxy_logging_obj.alerting_handler(
message=test_message, level="Low", alert_type=alert_type
message="This is a test slack alert message",
level="Low",
alert_type="budget_alerts",
)
return {
"status": "success",
"message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
}
else:
await proxy_logging_obj.alerting_handler(
message="This is a test slack alert message",
level="Low",
alert_type="budget_alerts",
raise HTTPException(
status_code=422,
detail={
"error": '"{}" not in proxy config: general_settings. Unable to test this.'.format(
service
)
},
)
return {
"status": "success",
"message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
}
else:
raise HTTPException(
status_code=422,
detail={
"error": '"slack" not in proxy config: general_settings. Unable to test this.'
},
)
except Exception as e:
if isinstance(e, HTTPException):
raise ProxyException(

1
litellm/utils.py
View file

@ -2130,7 +2130,6 @@ class Logging:
self.redact_message_input_output_from_logging(result=result)
print_verbose(f"Async success callbacks: {callbacks}")
for callback in callbacks:
# check if callback can run for this request
litellm_params = self.model_call_details.get("litellm_params", {})

127
ui/litellm-dashboard/src/components/settings.tsx
View file

@ -38,6 +38,7 @@ interface AlertingVariables {
LANGFUSE_PUBLIC_KEY: string | null,
LANGFUSE_SECRET_KEY: string | null,
LANGFUSE_HOST: string | null
OPENMETER_API_KEY: string | null
}
interface AlertingObject {
@ -45,12 +46,45 @@ interface AlertingObject {
variables: AlertingVariables
}
const defaultLoggingObject: AlertingObject[] = [
{
"name": "slack",
"variables": {
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null,
"SLACK_WEBHOOK_URL": null
}
},
{
"name": "langfuse",
"variables": {
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null,
"SLACK_WEBHOOK_URL": null
}
},
{
"name": "openmeter",
"variables": {
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null,
"SLACK_WEBHOOK_URL": null
}
}
]
const Settings: React.FC<SettingsPageProps> = ({
accessToken,
userRole,
userID,
}) => {
const [callbacks, setCallbacks] = useState<any[]>([]);
const [callbacks, setCallbacks] = useState<AlertingObject[]>(defaultLoggingObject);
const [alerts, setAlerts] = useState<any[]>([]);
const [isModalVisible, setIsModalVisible] = useState(false);
const [form] = Form.useForm();
@ -81,8 +115,19 @@ const Settings: React.FC<SettingsPageProps> = ({
}
getCallbacksCall(accessToken, userID, userRole).then((data) => {
console.log("callbacks", data);
let callbacks_data = data.callbacks;
setCallbacks(callbacks_data);
let updatedCallbacks: any[] = defaultLoggingObject;
updatedCallbacks = updatedCallbacks.map((item: any) => {
const callback = data.callbacks.find((cb: any) => cb.name === item.name);
if (callback) {
return { ...item, variables: { ...item.variables, ...callback.variables } };
} else {
return item;
}
});
setCallbacks(updatedCallbacks)
// setCallbacks(callbacks_data);
let alerts_data = data.alerts;
console.log("alerts_data", alerts_data);
@ -175,6 +220,9 @@ const Settings: React.FC<SettingsPageProps> = ({
const payload = {
environment_variables: updatedVariables,
litellm_settings: {
"success_callback": [callback.name]
}
};
try {
@ -212,7 +260,8 @@ const Settings: React.FC<SettingsPageProps> = ({
"SLACK_WEBHOOK_URL": null,
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": values.langfusePublicKey,
"LANGFUSE_SECRET_KEY": values.langfusePrivateKey
"LANGFUSE_SECRET_KEY": values.langfusePrivateKey,
OPENMETER_API_KEY: null
}
}
// add langfuse to callbacks
@ -239,10 +288,34 @@ const Settings: React.FC<SettingsPageProps> = ({
"SLACK_WEBHOOK_URL": values.slackWebhookUrl,
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null
}
}
setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
} else if (values.callback == "openmeter") {
console.log(`values.openMeterApiKey: ${values.openMeterApiKey}`)
payload = {
environment_variables: {
OPENMETER_API_KEY: values.openMeterApiKey,
},
litellm_settings: {
success_callback: [values.callback]
}
};
setCallbacksCall(accessToken, payload);
let newCallback: AlertingObject = {
"name": values.callback,
"variables": {
"SLACK_WEBHOOK_URL": null,
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
OPENMETER_API_KEY: values.openMeterAPIKey
}
}
// add langfuse to callbacks
setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
} else {
payload = {
error: 'Invalid callback value'
@ -282,24 +355,24 @@ const Settings: React.FC<SettingsPageProps> = ({
</TableRow>
</TableHead>
<TableBody>
{callbacks.map((callback, index) => (
{callbacks.filter((callback) => callback.name !== "slack").map((callback, index) => (
<TableRow key={index}>
<TableCell>
<Badge color="emerald">{callback.name}</Badge>
</TableCell>
<TableCell>
<ul>
{Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => (
<li key={key}>
<Text className="mt-2">{key}</Text>
{key === "LANGFUSE_HOST" ? (
<p>default value=https://cloud.langfuse.com</p>
) : (
<div></div>
)}
<TextInput name={key} defaultValue={value as string} type="password" />
</li>
))}
{Object.entries(callback.variables ?? {}).filter(([key, value]) => key.toLowerCase().includes(callback.name)).map(([key, value]) => (
<li key={key}>
<Text className="mt-2">{key}</Text>
{key === "LANGFUSE_HOST" ? (
<p>default value=https://cloud.langfuse.com</p>
) : (
<div></div>
)}
<TextInput name={key} defaultValue={value as string} type="password" />
</li>
))}
</ul>
<Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
Save Changes
@ -312,9 +385,6 @@ const Settings: React.FC<SettingsPageProps> = ({
))}
</TableBody>
</Table>
<Button size="xs" className="mt-2" onClick={handleAddCallback}>
Add Callback
</Button>
</Card>
</TabPanel>
@ -392,9 +462,10 @@ const Settings: React.FC<SettingsPageProps> = ({
>
<Select onChange={handleCallbackChange}>
<Select.Option value="langfuse">langfuse</Select.Option>
<Select.Option value="openmeter">openmeter</Select.Option>
</Select>
</Form.Item>
{selectedCallback === 'langfuse' && (
<>
<Form.Item
@ -419,6 +490,20 @@ const Settings: React.FC<SettingsPageProps> = ({
</>
)}
{
selectedCallback == "openmeter" && <>
<Form.Item
label="OPENMETER_API_KEY"
name="openMeterApiKey"
rules={[
{ required: true, message: "Please enter the openmeter api key" },
]}
>
<TextInput type="password"/>
</Form.Item>
</>
}
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Save</Button2>
</div>