phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(feat) working semantic cache on proxy

Browse source
This commit is contained in:
ishaan-jaff 2024-02-06 08:55:25 -08:00
parent a3b1e3bc84
commit 70a895329e
1 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/proxy/proxy_config.yaml
View file

@ -73,7 +73,12 @@ litellm_settings:
max_budget: 1.5000 max_budget: 1.5000
models: ["azure-gpt-3.5"] models: ["azure-gpt-3.5"]
duration: None duration: None
# cache: True cache: True # set cache responses to True
cache_params:
type: "redis-semantic"
similarity_threshold: 0.8
redis_semantic_cache_use_async: True
# cache: True
# setting callback class # setting callback class
# callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] # callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance]