phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix anthropic json location bug

Browse source
This commit is contained in:
Krrish Dholakia 2023-09-22 15:21:28 -07:00
parent 9ef0ad9e66
commit e1d0c6096c
4 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/utils.py
View file

@ -8,6 +8,7 @@ import tiktoken
import uuid
import aiohttp
from tokenizers import Tokenizer
import pkg_resources
encoding = tiktoken.get_encoding("cl100k_base")
import importlib.metadata
from .integrations.traceloop import TraceloopLogger
@ -723,7 +724,8 @@ def token_counter(model="", text=None, messages: Optional[List] = None):
# anthropic
elif model in litellm.anthropic_models:
# Read the JSON file
with open('../llms/tokenizers/anthropic_tokenizer.json', 'r') as f:
filename = pkg_resources.resource_filename(__name__, 'llms/tokenizers/anthropic_tokenizer.json')
with open(filename, 'r') as f:
json_data = json.load(f)
# Decode the JSON data from utf-8
json_data_decoded = json.dumps(json_data, ensure_ascii=False)