phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(test_async_fn.py): refactor for testing

Browse source
This commit is contained in:
Krrish Dholakia 2023-10-24 12:46:30 -07:00
parent b7a023a82b
commit 2c371bb8d1
2 changed files with 34 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/utils.py
View file

@ -20,7 +20,6 @@ import aiohttp
import logging
import asyncio
from tokenizers import Tokenizer
import pkg_resources
from dataclasses import (
dataclass,
field,
@ -875,6 +874,7 @@ def get_replicate_completion_pricing(completion_response=None, total_time=0.0):
def _select_tokenizer(model: str):
# cohere
import pkg_resources
if model in litellm.cohere_models:
tokenizer = Tokenizer.from_pretrained("Cohere/command-nightly")
return {"type": "huggingface_tokenizer", "tokenizer": tokenizer}