phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(utils.py): use local tiktoken copy

Browse source
This commit is contained in:
Krrish Dholakia 2023-12-28 11:22:17 +05:30
parent 7751b83100
commit c4fc28ab0d
3 changed files with 100262 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

100256
litellm/llms/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 Normal file
View file

File diff suppressed because it is too large Load diff

2
litellm/tests/test_token_counter.py
View file

@ -46,7 +46,7 @@ def test_token_counter_normal_plus_function_calling():
pytest.fail(f"An exception occurred - {str(e)}") pytest.fail(f"An exception occurred - {str(e)}")
test_token_counter_normal_plus_function_calling() # test_token_counter_normal_plus_function_calling()
def test_tokenizers(): def test_tokenizers():

5
litellm/utils.py
View file

@ -27,7 +27,12 @@ from dataclasses import (
dataclass, dataclass,
field, field,
) # for storing API inputs, outputs, and metadata ) # for storing API inputs, outputs, and metadata
import pkg_resources
filename = pkg_resources.resource_filename(__name__, "llms/tokenizers")
os.environ[
"TIKTOKEN_CACHE_DIR"
] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
encoding = tiktoken.get_encoding("cl100k_base") encoding = tiktoken.get_encoding("cl100k_base")
import importlib.metadata import importlib.metadata
from .integrations.traceloop import TraceloopLogger from .integrations.traceloop import TraceloopLogger