phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

build(ui): enable adding openmeter via proxy ui

Browse source
This commit is contained in:
Krrish Dholakia 2024-05-01 21:16:23 -07:00
parent dbf999f56e
commit cdd3e1eef3
5 changed files with 214 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/my-website/docs/proxy/logging.md
View file

@ -600,7 +600,7 @@ Bill customers according to their LLM API usage with [OpenMeter](../observabilit
```bash ```bash
# from https://openmeter.cloud # from https://openmeter.cloud
export OPENMETER_API_ENDPOINT="" export OPENMETER_API_ENDPOINT="" # defaults to https://openmeter.cloud
export OPENMETER_API_KEY="" export OPENMETER_API_KEY=""
``` ```

11
litellm/integrations/openmeter.py
View file

@ -38,9 +38,6 @@ class OpenMeterLogger(CustomLogger):
in the environment in the environment
""" """
missing_keys = [] missing_keys = []
if litellm.get_secret("OPENMETER_API_ENDPOINT", None) is None:
missing_keys.append("OPENMETER_API_ENDPOINT")
if litellm.get_secret("OPENMETER_API_KEY", None) is None: if litellm.get_secret("OPENMETER_API_KEY", None) is None:
missing_keys.append("OPENMETER_API_KEY") missing_keys.append("OPENMETER_API_KEY")
@ -74,7 +71,9 @@ class OpenMeterLogger(CustomLogger):
} }
def log_success_event(self, kwargs, response_obj, start_time, end_time): def log_success_event(self, kwargs, response_obj, start_time, end_time):
_url = litellm.get_secret("OPENMETER_API_ENDPOINT") _url = litellm.get_secret(
"OPENMETER_API_ENDPOINT", default_value="https://openmeter.cloud"
)
if _url.endswith("/"): if _url.endswith("/"):
_url += "api/v1/events" _url += "api/v1/events"
else: else:
@ -93,7 +92,9 @@ class OpenMeterLogger(CustomLogger):
) )
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
_url = litellm.get_secret("OPENMETER_API_ENDPOINT") _url = litellm.get_secret(
"OPENMETER_API_ENDPOINT", default_value="https://openmeter.cloud"
)
if _url.endswith("/"): if _url.endswith("/"):
_url += "api/v1/events" _url += "api/v1/events"
else: else:

161
litellm/proxy/proxy_server.py
View file

@ -2693,9 +2693,10 @@ class ProxyConfig:
environment_variables = config_data.get("environment_variables", {}) environment_variables = config_data.get("environment_variables", {})
for k, v in environment_variables.items(): for k, v in environment_variables.items():
try: try:
decoded_b64 = base64.b64decode(v) if v is not None:
value = decrypt_value(value=decoded_b64, master_key=master_key) # type: ignore decoded_b64 = base64.b64decode(v)
os.environ[k] = value value = decrypt_value(value=decoded_b64, master_key=master_key) # type: ignore
os.environ[k] = value
except Exception as e: except Exception as e:
verbose_proxy_logger.error( verbose_proxy_logger.error(
"Error setting env variable: %s - %s", k, str(e) "Error setting env variable: %s - %s", k, str(e)
@ -8644,6 +8645,13 @@ async def update_config(config_info: ConfigYAML):
_existing_settings = config["general_settings"] _existing_settings = config["general_settings"]
for k, v in updated_general_settings.items(): for k, v in updated_general_settings.items():
# overwrite existing settings with updated values # overwrite existing settings with updated values
if k == "alert_to_webhook_url":
# check if slack is already enabled. if not, enable it
if "slack" not in _existing_settings:
if "alerting" not in _existing_settings:
_existing_settings["alerting"] = ["slack"]
elif isinstance(_existing_settings["alerting"], list):
_existing_settings["alerting"].append("slack")
_existing_settings[k] = v _existing_settings[k] = v
config["general_settings"] = _existing_settings config["general_settings"] = _existing_settings
@ -8758,7 +8766,25 @@ async def get_config():
""" """
for _callback in _success_callbacks: for _callback in _success_callbacks:
if _callback == "langfuse": if _callback == "openmeter":
env_vars = [
"OPENMETER_API_KEY",
]
env_vars_dict = {}
for _var in env_vars:
env_variable = environment_variables.get(_var, None)
if env_variable is None:
env_vars_dict[_var] = None
else:
# decode + decrypt the value
decoded_b64 = base64.b64decode(env_variable)
_decrypted_value = decrypt_value(
value=decoded_b64, master_key=master_key
)
env_vars_dict[_var] = _decrypted_value
_data_to_return.append({"name": _callback, "variables": env_vars_dict})
elif _callback == "langfuse":
_langfuse_vars = [ _langfuse_vars = [
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_PUBLIC_KEY",
"LANGFUSE_SECRET_KEY", "LANGFUSE_SECRET_KEY",
@ -8898,9 +8924,9 @@ async def test_endpoint(request: Request):
) )
async def health_services_endpoint( async def health_services_endpoint(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
service: Literal["slack_budget_alerts", "langfuse", "slack"] = fastapi.Query( service: Literal[
description="Specify the service being hit." "slack_budget_alerts", "langfuse", "slack", "openmeter"
), ] = fastapi.Query(description="Specify the service being hit."),
): ):
""" """
Hidden endpoint. Hidden endpoint.
@ -8914,7 +8940,7 @@ async def health_services_endpoint(
raise HTTPException( raise HTTPException(
status_code=400, detail={"error": "Service must be specified."} status_code=400, detail={"error": "Service must be specified."}
) )
if service not in ["slack_budget_alerts", "langfuse", "slack"]: if service not in ["slack_budget_alerts", "langfuse", "slack", "openmeter"]:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail={ detail={
@ -8922,6 +8948,18 @@ async def health_services_endpoint(
}, },
) )
if service == "openmeter":
_ = await litellm.acompletion(
model="openai/litellm-mock-response-model",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
user="litellm:/health/services",
mock_response="This is a mock response",
)
return {
"status": "success",
"message": "Mock LLM request made - check openmeter.",
}
if service == "langfuse": if service == "langfuse":
from litellm.integrations.langfuse import LangFuseLogger from litellm.integrations.langfuse import LangFuseLogger
@ -8938,63 +8976,66 @@ async def health_services_endpoint(
"message": "Mock LLM request made - check langfuse.", "message": "Mock LLM request made - check langfuse.",
} }
if "slack" in general_settings.get("alerting", []): if service == "slack" or service == "slack_budget_alerts":
# test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" if "slack" in general_settings.get("alerting", []):
# check if user has opted into unique_alert_webhooks # test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
if ( # check if user has opted into unique_alert_webhooks
proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url if (
is not None proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
): is not None
for ( ):
alert_type for (
) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url: alert_type
""" ) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url:
"llm_exceptions", """
"llm_too_slow", "llm_exceptions",
"llm_requests_hanging", "llm_too_slow",
"budget_alerts", "llm_requests_hanging",
"db_exceptions", "budget_alerts",
""" "db_exceptions",
# only test alert if it's in active alert types """
if ( # only test alert if it's in active alert types
proxy_logging_obj.slack_alerting_instance.alert_types if (
is not None proxy_logging_obj.slack_alerting_instance.alert_types
and alert_type is not None
not in proxy_logging_obj.slack_alerting_instance.alert_types and alert_type
): not in proxy_logging_obj.slack_alerting_instance.alert_types
continue ):
test_message = "default test message" continue
if alert_type == "llm_exceptions": test_message = "default test message"
test_message = f"LLM Exception test alert" if alert_type == "llm_exceptions":
elif alert_type == "llm_too_slow": test_message = f"LLM Exception test alert"
test_message = f"LLM Too Slow test alert" elif alert_type == "llm_too_slow":
elif alert_type == "llm_requests_hanging": test_message = f"LLM Too Slow test alert"
test_message = f"LLM Requests Hanging test alert" elif alert_type == "llm_requests_hanging":
elif alert_type == "budget_alerts": test_message = f"LLM Requests Hanging test alert"
test_message = f"Budget Alert test alert" elif alert_type == "budget_alerts":
elif alert_type == "db_exceptions": test_message = f"Budget Alert test alert"
test_message = f"DB Exception test alert" elif alert_type == "db_exceptions":
test_message = f"DB Exception test alert"
await proxy_logging_obj.alerting_handler(
message=test_message, level="Low", alert_type=alert_type
)
else:
await proxy_logging_obj.alerting_handler( await proxy_logging_obj.alerting_handler(
message=test_message, level="Low", alert_type=alert_type message="This is a test slack alert message",
level="Low",
alert_type="budget_alerts",
) )
return {
"status": "success",
"message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
}
else: else:
await proxy_logging_obj.alerting_handler( raise HTTPException(
message="This is a test slack alert message", status_code=422,
level="Low", detail={
alert_type="budget_alerts", "error": '"{}" not in proxy config: general_settings. Unable to test this.'.format(
service
)
},
) )
return {
"status": "success",
"message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
}
else:
raise HTTPException(
status_code=422,
detail={
"error": '"slack" not in proxy config: general_settings. Unable to test this.'
},
)
except Exception as e: except Exception as e:
if isinstance(e, HTTPException): if isinstance(e, HTTPException):
raise ProxyException( raise ProxyException(

1
litellm/utils.py
View file

@ -2130,7 +2130,6 @@ class Logging:
self.redact_message_input_output_from_logging(result=result) self.redact_message_input_output_from_logging(result=result)
print_verbose(f"Async success callbacks: {callbacks}")
for callback in callbacks: for callback in callbacks:
# check if callback can run for this request # check if callback can run for this request
litellm_params = self.model_call_details.get("litellm_params", {}) litellm_params = self.model_call_details.get("litellm_params", {})

125
ui/litellm-dashboard/src/components/settings.tsx
View file

@ -38,6 +38,7 @@ interface AlertingVariables {
LANGFUSE_PUBLIC_KEY: string | null, LANGFUSE_PUBLIC_KEY: string | null,
LANGFUSE_SECRET_KEY: string | null, LANGFUSE_SECRET_KEY: string | null,
LANGFUSE_HOST: string | null LANGFUSE_HOST: string | null
OPENMETER_API_KEY: string | null
} }
interface AlertingObject { interface AlertingObject {
@ -45,12 +46,45 @@ interface AlertingObject {
variables: AlertingVariables variables: AlertingVariables
} }
const defaultLoggingObject: AlertingObject[] = [
{
"name": "slack",
"variables": {
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null,
"SLACK_WEBHOOK_URL": null
}
},
{
"name": "langfuse",
"variables": {
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null,
"SLACK_WEBHOOK_URL": null
}
},
{
"name": "openmeter",
"variables": {
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null,
"SLACK_WEBHOOK_URL": null
}
}
]
const Settings: React.FC<SettingsPageProps> = ({ const Settings: React.FC<SettingsPageProps> = ({
accessToken, accessToken,
userRole, userRole,
userID, userID,
}) => { }) => {
const [callbacks, setCallbacks] = useState<any[]>([]); const [callbacks, setCallbacks] = useState<AlertingObject[]>(defaultLoggingObject);
const [alerts, setAlerts] = useState<any[]>([]); const [alerts, setAlerts] = useState<any[]>([]);
const [isModalVisible, setIsModalVisible] = useState(false); const [isModalVisible, setIsModalVisible] = useState(false);
const [form] = Form.useForm(); const [form] = Form.useForm();
@ -81,8 +115,19 @@ const Settings: React.FC<SettingsPageProps> = ({
} }
getCallbacksCall(accessToken, userID, userRole).then((data) => { getCallbacksCall(accessToken, userID, userRole).then((data) => {
console.log("callbacks", data); console.log("callbacks", data);
let callbacks_data = data.callbacks; let updatedCallbacks: any[] = defaultLoggingObject;
setCallbacks(callbacks_data);
updatedCallbacks = updatedCallbacks.map((item: any) => {
const callback = data.callbacks.find((cb: any) => cb.name === item.name);
if (callback) {
return { ...item, variables: { ...item.variables, ...callback.variables } };
} else {
return item;
}
});
setCallbacks(updatedCallbacks)
// setCallbacks(callbacks_data);
let alerts_data = data.alerts; let alerts_data = data.alerts;
console.log("alerts_data", alerts_data); console.log("alerts_data", alerts_data);
@ -175,6 +220,9 @@ const Settings: React.FC<SettingsPageProps> = ({
const payload = { const payload = {
environment_variables: updatedVariables, environment_variables: updatedVariables,
litellm_settings: {
"success_callback": [callback.name]
}
}; };
try { try {
@ -212,7 +260,8 @@ const Settings: React.FC<SettingsPageProps> = ({
"SLACK_WEBHOOK_URL": null, "SLACK_WEBHOOK_URL": null,
"LANGFUSE_HOST": null, "LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": values.langfusePublicKey, "LANGFUSE_PUBLIC_KEY": values.langfusePublicKey,
"LANGFUSE_SECRET_KEY": values.langfusePrivateKey "LANGFUSE_SECRET_KEY": values.langfusePrivateKey,
OPENMETER_API_KEY: null
} }
} }
// add langfuse to callbacks // add langfuse to callbacks
@ -239,10 +288,34 @@ const Settings: React.FC<SettingsPageProps> = ({
"SLACK_WEBHOOK_URL": values.slackWebhookUrl, "SLACK_WEBHOOK_URL": values.slackWebhookUrl,
"LANGFUSE_HOST": null, "LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null, "LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null "LANGFUSE_SECRET_KEY": null,
"OPENMETER_API_KEY": null
} }
} }
setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]); setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
} else if (values.callback == "openmeter") {
console.log(`values.openMeterApiKey: ${values.openMeterApiKey}`)
payload = {
environment_variables: {
OPENMETER_API_KEY: values.openMeterApiKey,
},
litellm_settings: {
success_callback: [values.callback]
}
};
setCallbacksCall(accessToken, payload);
let newCallback: AlertingObject = {
"name": values.callback,
"variables": {
"SLACK_WEBHOOK_URL": null,
"LANGFUSE_HOST": null,
"LANGFUSE_PUBLIC_KEY": null,
"LANGFUSE_SECRET_KEY": null,
OPENMETER_API_KEY: values.openMeterAPIKey
}
}
// add langfuse to callbacks
setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
} else { } else {
payload = { payload = {
error: 'Invalid callback value' error: 'Invalid callback value'
@ -282,24 +355,24 @@ const Settings: React.FC<SettingsPageProps> = ({
</TableRow> </TableRow>
</TableHead> </TableHead>
<TableBody> <TableBody>
{callbacks.map((callback, index) => ( {callbacks.filter((callback) => callback.name !== "slack").map((callback, index) => (
<TableRow key={index}> <TableRow key={index}>
<TableCell> <TableCell>
<Badge color="emerald">{callback.name}</Badge> <Badge color="emerald">{callback.name}</Badge>
</TableCell> </TableCell>
<TableCell> <TableCell>
<ul> <ul>
{Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => ( {Object.entries(callback.variables ?? {}).filter(([key, value]) => key.toLowerCase().includes(callback.name)).map(([key, value]) => (
<li key={key}> <li key={key}>
<Text className="mt-2">{key}</Text> <Text className="mt-2">{key}</Text>
{key === "LANGFUSE_HOST" ? ( {key === "LANGFUSE_HOST" ? (
<p>default value=https://cloud.langfuse.com</p> <p>default value=https://cloud.langfuse.com</p>
) : ( ) : (
<div></div> <div></div>
)} )}
<TextInput name={key} defaultValue={value as string} type="password" /> <TextInput name={key} defaultValue={value as string} type="password" />
</li> </li>
))} ))}
</ul> </ul>
<Button className="mt-2" onClick={() => handleSaveChanges(callback)}> <Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
Save Changes Save Changes
@ -312,9 +385,6 @@ const Settings: React.FC<SettingsPageProps> = ({
))} ))}
</TableBody> </TableBody>
</Table> </Table>
<Button size="xs" className="mt-2" onClick={handleAddCallback}>
Add Callback
</Button>
</Card> </Card>
</TabPanel> </TabPanel>
@ -392,6 +462,7 @@ const Settings: React.FC<SettingsPageProps> = ({
> >
<Select onChange={handleCallbackChange}> <Select onChange={handleCallbackChange}>
<Select.Option value="langfuse">langfuse</Select.Option> <Select.Option value="langfuse">langfuse</Select.Option>
<Select.Option value="openmeter">openmeter</Select.Option>
</Select> </Select>
</Form.Item> </Form.Item>
@ -419,6 +490,20 @@ const Settings: React.FC<SettingsPageProps> = ({
</> </>
)} )}
{
selectedCallback == "openmeter" && <>
<Form.Item
label="OPENMETER_API_KEY"
name="openMeterApiKey"
rules={[
{ required: true, message: "Please enter the openmeter api key" },
]}
>
<TextInput type="password"/>
</Form.Item>
</>
}
<div style={{ textAlign: "right", marginTop: "10px" }}> <div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Save</Button2> <Button2 htmlType="submit">Save</Button2>
</div> </div>