forked from phoenix/litellm-mirror
(docs) s3 cache
This commit is contained in:
parent
d45101b652
commit
344e8d8508
1 changed files with 81 additions and 29 deletions
|
@ -1,18 +1,29 @@
|
||||||
|
import Tabs from '@theme/Tabs';
|
||||||
|
import TabItem from '@theme/TabItem';
|
||||||
|
|
||||||
# Redis Cache, s3 Cache
|
# Redis Cache, s3 Cache
|
||||||
|
|
||||||
[**See Code**](https://github.com/BerriAI/litellm/blob/4d7ff1b33b9991dcf38d821266290631d9bcd2dd/litellm/caching.py#L71)
|
[**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching.py)
|
||||||
|
|
||||||
|
## Initialize Cache - In Memory, Redis, s3 Bucket
|
||||||
|
|
||||||
|
|
||||||
|
<Tabs>
|
||||||
|
|
||||||
|
<TabItem value="redis" label="redis-cache">
|
||||||
|
|
||||||
### Pre-requisites
|
|
||||||
Install redis
|
Install redis
|
||||||
```shell
|
```shell
|
||||||
pip install redis
|
pip install redis
|
||||||
```
|
```
|
||||||
|
|
||||||
For the hosted version you can setup your own Redis DB here: https://app.redislabs.com/
|
For the hosted version you can setup your own Redis DB here: https://app.redislabs.com/
|
||||||
### Quick Start
|
### Quick Start
|
||||||
```python
|
```python
|
||||||
import litellm
|
import litellm
|
||||||
from litellm import completion
|
from litellm import completion
|
||||||
from litellm.caching import Cache
|
from litellm.caching import Cache
|
||||||
|
|
||||||
litellm.cache = Cache(type="redis", host=<host>, port=<port>, password=<password>)
|
litellm.cache = Cache(type="redis", host=<host>, port=<port>, password=<password>)
|
||||||
|
|
||||||
# Make completion calls
|
# Make completion calls
|
||||||
|
@ -28,6 +39,48 @@ response2 = completion(
|
||||||
# response1 == response2, response 1 is cached
|
# response1 == response2, response 1 is cached
|
||||||
```
|
```
|
||||||
|
|
||||||
|
</TabItem>
|
||||||
|
|
||||||
|
|
||||||
|
<TabItem value="s3" label="s3-cache">
|
||||||
|
|
||||||
|
Install boto3
|
||||||
|
```shell
|
||||||
|
pip install boto3
|
||||||
|
```
|
||||||
|
|
||||||
|
Set AWS environment variables
|
||||||
|
|
||||||
|
```shell
|
||||||
|
AWS_ACCESS_KEY_ID = "AKI*******"
|
||||||
|
AWS_SECRET_ACCESS_KEY = "WOl*****"
|
||||||
|
```
|
||||||
|
### Quick Start
|
||||||
|
```python
|
||||||
|
import litellm
|
||||||
|
from litellm import completion
|
||||||
|
from litellm.caching import Cache
|
||||||
|
|
||||||
|
# pass s3-bucket name
|
||||||
|
litellm.cache = Cache(type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2")
|
||||||
|
|
||||||
|
# Make completion calls
|
||||||
|
response1 = completion(
|
||||||
|
model="gpt-3.5-turbo",
|
||||||
|
messages=[{"role": "user", "content": "Tell me a joke."}]
|
||||||
|
)
|
||||||
|
response2 = completion(
|
||||||
|
model="gpt-3.5-turbo",
|
||||||
|
messages=[{"role": "user", "content": "Tell me a joke."}]
|
||||||
|
)
|
||||||
|
|
||||||
|
# response1 == response2, response 1 is cached
|
||||||
|
```
|
||||||
|
|
||||||
|
</TabItem>
|
||||||
|
|
||||||
|
</Tabs>
|
||||||
|
|
||||||
## Cache Context Manager - Enable, Disable, Update Cache
|
## Cache Context Manager - Enable, Disable, Update Cache
|
||||||
Use the context manager for easily enabling, disabling & updating the litellm cache
|
Use the context manager for easily enabling, disabling & updating the litellm cache
|
||||||
|
|
||||||
|
@ -103,35 +156,34 @@ litellm.cache = cache # set litellm.cache to your cache
|
||||||
|
|
||||||
## Cache Initialization Parameters
|
## Cache Initialization Parameters
|
||||||
|
|
||||||
#### `type` (str, optional)
|
```python
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
type: Optional[Literal["local", "redis", "s3"]] = "local",
|
||||||
|
supported_call_types: Optional[
|
||||||
|
List[Literal["completion", "acompletion", "embedding", "aembedding"]]
|
||||||
|
] = ["completion", "acompletion", "embedding", "aembedding"], # A list of litellm call types to cache for. Defaults to caching for all litellm call types.
|
||||||
|
|
||||||
|
# redis cache params
|
||||||
|
host: Optional[str] = None,
|
||||||
|
port: Optional[str] = None,
|
||||||
|
password: Optional[str] = None,
|
||||||
|
|
||||||
The type of cache to initialize. It can be either "local" or "redis". Defaults to "local".
|
|
||||||
|
|
||||||
#### `host` (str, optional)
|
|
||||||
|
|
||||||
The host address for the Redis cache. This parameter is required if the `type` is set to "redis".
|
|
||||||
|
|
||||||
#### `port` (int, optional)
|
|
||||||
|
|
||||||
The port number for the Redis cache. This parameter is required if the `type` is set to "redis".
|
|
||||||
|
|
||||||
#### `password` (str, optional)
|
|
||||||
|
|
||||||
The password for the Redis cache. This parameter is required if the `type` is set to "redis".
|
|
||||||
|
|
||||||
#### `supported_call_types` (list, optional)
|
|
||||||
|
|
||||||
A list of call types to cache for. Defaults to caching for all call types. The available call types are:
|
|
||||||
|
|
||||||
- "completion"
|
|
||||||
- "acompletion"
|
|
||||||
- "embedding"
|
|
||||||
- "aembedding"
|
|
||||||
|
|
||||||
#### `**kwargs` (additional keyword arguments)
|
|
||||||
|
|
||||||
Additional keyword arguments are accepted for the initialization of the Redis cache using the `redis.Redis()` constructor. These arguments allow you to fine-tune the Redis cache configuration based on your specific needs.
|
|
||||||
|
|
||||||
|
# s3 Bucket, boto3 configuration
|
||||||
|
s3_bucket_name: Optional[str] = None,
|
||||||
|
s3_region_name: Optional[str] = None,
|
||||||
|
s3_api_version: Optional[str] = None,
|
||||||
|
s3_use_ssl: Optional[bool] = True,
|
||||||
|
s3_verify: Optional[Union[bool, str]] = None,
|
||||||
|
s3_endpoint_url: Optional[str] = None,
|
||||||
|
s3_aws_access_key_id: Optional[str] = None,
|
||||||
|
s3_aws_secret_access_key: Optional[str] = None,
|
||||||
|
s3_aws_session_token: Optional[str] = None,
|
||||||
|
s3_config: Optional[Any] = None,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
```
|
||||||
|
|
||||||
## Logging
|
## Logging
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue