phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(docs) use s3 Cache on litellm proxy

Browse source
This commit is contained in:
ishaan-jaff 2024-01-03 17:41:26 +05:30
parent f9139e05e8
commit 30c6d164d2
2 changed files with 67 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

37
docs/my-website/docs/proxy/caching.md
View file

@ -9,8 +9,13 @@ LiteLLM supports:
- Redis Cache - Redis Cache
- s3 Bucket Cache - s3 Bucket Cache
## Quick Start ## Quick Start - Redis, s3 Cache
<Tabs>
<TabItem value="redis" label="redis cache">
Caching can be enabled by adding the `cache` key in the `config.yaml` Caching can be enabled by adding the `cache` key in the `config.yaml`
### Step 1: Add `cache` to the config.yaml ### Step 1: Add `cache` to the config.yaml
```yaml ```yaml
model_list: model_list:
@ -48,7 +53,37 @@ REDIS_<redis-kwarg-name> = ""
```shell ```shell
$ litellm --config /path/to/config.yaml $ litellm --config /path/to/config.yaml
``` ```
</TabItem>
<TabItem value="s3" label="s3 cache">
### Step 1: Add `cache` to the config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
- model_name: text-embedding-ada-002
litellm_params:
model: text-embedding-ada-002
litellm_settings:
set_verbose: True
cache: True # set cache responses to True
cache_params: # set cache params for s3
type: s3
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: your_access_key # AWS Access Key ID for S3
s3_aws_secret_access_key: your_secret_key # AWS Secret Access Key for S3
```
### Step 2: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
</TabItem>
</Tabs>
## Using Caching - /chat/completions ## Using Caching - /chat/completions

62
litellm/proxy/proxy_server.py
View file

@ -544,46 +544,46 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: str):
print(f"{blue_color_code}\nSetting Cache on Proxy") # noqa print(f"{blue_color_code}\nSetting Cache on Proxy") # noqa
from litellm.caching import Cache from litellm.caching import Cache
if isinstance(value, dict): cache_params = {}
cache_type = value.get("type", "redis")
else:
cache_type = "redis" # default to using redis on cache
cache_responses = True
cache_host = litellm.get_secret("REDIS_HOST", None)
cache_port = litellm.get_secret("REDIS_PORT", None)
cache_password = litellm.get_secret("REDIS_PASSWORD", None)
cache_params = {
"type": cache_type,
"host": cache_host,
"port": cache_port,
"password": cache_password,
}
if "cache_params" in litellm_settings: if "cache_params" in litellm_settings:
cache_params_in_config = litellm_settings["cache_params"] cache_params_in_config = litellm_settings["cache_params"]
# overwrie cache_params with cache_params_in_config # overwrie cache_params with cache_params_in_config
cache_params.update(cache_params_in_config) cache_params.update(cache_params_in_config)
# Assuming cache_type, cache_host, cache_port, and cache_password are strings cache_type = cache_params.get("type", "redis")
print( # noqa
f"{blue_color_code}Cache Type:{reset_color_code} {cache_type}" print_verbose(f"passed cache type={cache_type}")
) # noqa
print( # noqa if cache_type == "redis":
f"{blue_color_code}Cache Host:{reset_color_code} {cache_host}" cache_host = litellm.get_secret("REDIS_HOST", None)
) # noqa cache_port = litellm.get_secret("REDIS_PORT", None)
print( # noqa cache_password = litellm.get_secret("REDIS_PASSWORD", None)
f"{blue_color_code}Cache Port:{reset_color_code} {cache_port}"
) # noqa cache_params = {
print( # noqa "type": cache_type,
f"{blue_color_code}Cache Password:{reset_color_code} {cache_password}" "host": cache_host,
) "port": cache_port,
print() # noqa "password": cache_password,
}
# Assuming cache_type, cache_host, cache_port, and cache_password are strings
print( # noqa
f"{blue_color_code}Cache Type:{reset_color_code} {cache_type}"
) # noqa
print( # noqa
f"{blue_color_code}Cache Host:{reset_color_code} {cache_host}"
) # noqa
print( # noqa
f"{blue_color_code}Cache Port:{reset_color_code} {cache_port}"
) # noqa
print( # noqa
f"{blue_color_code}Cache Password:{reset_color_code} {cache_password}"
)
print() # noqa
## to pass a complete url, or set ssl=True, etc. just set it as `os.environ[REDIS_URL] = <your-redis-url>`, _redis.py checks for REDIS specific environment variables ## to pass a complete url, or set ssl=True, etc. just set it as `os.environ[REDIS_URL] = <your-redis-url>`, _redis.py checks for REDIS specific environment variables
litellm.cache = Cache(**cache_params) litellm.cache = Cache(**cache_params)
print( # noqa print( # noqa
f"{blue_color_code}Set Cache on LiteLLM Proxy: {litellm.cache.cache}{reset_color_code} {cache_password}" f"{blue_color_code}Set Cache on LiteLLM Proxy: {vars(litellm.cache.cache)}{reset_color_code}"
) )
elif key == "callbacks": elif key == "callbacks":
litellm.callbacks = [ litellm.callbacks = [