phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
sumanth 2023-12-19 13:15:48 +05:30
parent 121149fed4
commit 64dfd0d9a6
2 changed files with 53 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

41
litellm/tests/test_router_caching.py
View file

@ -124,4 +124,43 @@ async def test_acompletion_caching_on_router_caching_groups():
pass
except Exception as e:
traceback.print_exc()
pytest.fail(f"Error occurred: {e}")
pytest.fail(f"Error occurred: {e}")
def test_usage_based_routing_completion():
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "gpt-3.5-turbo-0301",
"api_key": os.getenv("OPENAI_API_KEY"),
"custom_llm_provider": "Custom-LLM"
},
"tpm": 10000,
"rpm": 5
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "gpt-3.5-turbo-0301",
"api_key": os.getenv("OPENAI_API_KEY"),
},
"tpm": 10000,
"rpm": 5
}
]
router = Router(model_list= model_list,
routing_strategy= "usage-based-routing",
set_verbose= False)
max_requests = 5
while max_requests > 0:
try:
router.completion(
model='gpt-3.5-turbo',
messages=[{"content": "write a one sentence poem.", "role": "user"}],
)
except ValueError as e:
traceback.print_exc()
pytest.fail(f"Error occurred: {e}")
finally:
max_requests -= 1
router.reset()