mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-26 11:14:04 +00:00
dec53961f7
* fix(caching.py): set ttl for async_increment cache fixes issue where ttl for redis client was not being set on increment_cache Fixes https://github.com/BerriAI/litellm/issues/5609 * fix(caching.py): fix increment cache w/ ttl for sync increment cache on redis Fixes https://github.com/BerriAI/litellm/issues/5609 * fix(router.py): support adding retry policy + allowed fails policy via config.yaml * fix(router.py): don't cooldown single deployments No point, as there's no other deployment to loadbalance with. * fix(user_api_key_auth.py): support setting allowed email domains on jwt tokens Closes https://github.com/BerriAI/litellm/issues/5605 * docs(token_auth.md): add user upsert + allowed email domain to jwt auth docs * fix(litellm_pre_call_utils.py): fix dynamic key logging when team id is set Fixes issue where key logging would not be set if team metadata was not none * fix(secret_managers/main.py): load environment variables correctly Fixes issue where os.environ/ was not being loaded correctly * test(test_router.py): fix test * feat(spend_tracking_utils.py): support logging additional usage params - e.g. prompt caching values for deepseek * test: fix tests * test: fix test * test: fix test * test: fix test * test: fix test
118 lines
3 KiB
Python
118 lines
3 KiB
Python
import io
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.abspath("../.."))
|
|
|
|
import litellm
|
|
from litellm import completion
|
|
|
|
litellm.failure_callback = ["lunary"]
|
|
litellm.success_callback = ["lunary"]
|
|
litellm.set_verbose = True
|
|
|
|
|
|
def test_lunary_logging():
|
|
try:
|
|
response = completion(
|
|
model="gpt-3.5-turbo",
|
|
messages=[{"role": "user", "content": "what llm are u"}],
|
|
max_tokens=10,
|
|
temperature=0.2,
|
|
user="test-user",
|
|
)
|
|
print(response)
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
|
|
test_lunary_logging()
|
|
|
|
|
|
def test_lunary_template():
|
|
import lunary
|
|
|
|
try:
|
|
template = lunary.render_template("test-template", {"question": "Hello!"})
|
|
response = completion(**template)
|
|
print(response)
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
|
|
def test_lunary_logging_with_metadata():
|
|
try:
|
|
response = completion(
|
|
model="gpt-3.5-turbo",
|
|
messages=[{"role": "user", "content": "what llm are u"}],
|
|
max_tokens=10,
|
|
temperature=0.2,
|
|
metadata={
|
|
"run_name": "litellmRUN",
|
|
"project_name": "litellm-completion",
|
|
"tags": ["tag1", "tag2"],
|
|
},
|
|
)
|
|
print(response)
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
|
|
def test_lunary_with_tools():
|
|
import litellm
|
|
|
|
messages = [
|
|
{
|
|
"role": "user",
|
|
"content": "What's the weather like in San Francisco, Tokyo, and Paris?",
|
|
}
|
|
]
|
|
tools = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_current_weather",
|
|
"description": "Get the current weather in a given location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city and state, e.g. San Francisco, CA",
|
|
},
|
|
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
|
},
|
|
"required": ["location"],
|
|
},
|
|
},
|
|
}
|
|
]
|
|
|
|
response = litellm.completion(
|
|
model="gpt-3.5-turbo-1106",
|
|
messages=messages,
|
|
tools=tools,
|
|
tool_choice="auto", # auto is default, but we'll be explicit
|
|
)
|
|
|
|
response_message = response.choices[0].message
|
|
print("\nLLM Response:\n", response.choices[0].message)
|
|
|
|
|
|
def test_lunary_logging_with_streaming_and_metadata():
|
|
try:
|
|
response = completion(
|
|
model="gpt-3.5-turbo",
|
|
messages=[{"role": "user", "content": "what llm are u"}],
|
|
max_tokens=10,
|
|
temperature=0.2,
|
|
metadata={
|
|
"run_name": "litellmRUN",
|
|
"project_name": "litellm-completion",
|
|
},
|
|
stream=True,
|
|
)
|
|
for chunk in response:
|
|
continue
|
|
except Exception as e:
|
|
print(e)
|