phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) track cost for semantic_caching, place on langfuse trace

Browse source
This commit is contained in:
ishaan-jaff 2024-02-07 19:21:50 -08:00
parent 4cb7759fcd
commit 8425a8ba22
1 changed files with 13 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
litellm/caching.py
View file

@ -427,10 +427,16 @@ class RedisSemanticCache(BaseCache):
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
embedding_response = await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
metadata={
"user_api_key": user_api_key,
"semantic-cache-embedding": True,
"trace_id": kwargs.get("metadata", {}).get("trace_id", None),
},
)
else:
# convert to embedding
@ -476,13 +482,20 @@ class RedisSemanticCache(BaseCache):
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
embedding_response = await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
metadata={
"user_api_key": user_api_key,
"semantic-cache-embedding": True,
"trace_id": kwargs.get("metadata", {}).get("trace_id", None),
},
)
else:
# convert to embedding
user_api_key = kwargs["litellm_params"]["metadata"].get("user_api_key", "")
embedding_response = await litellm.aembedding(
model=self.embedding_model,
input=prompt,