forked from phoenix/litellm-mirror
docs(openai.md): fix docs to include example of calling openai on proxy
This commit is contained in:
parent
d3e61a0adc
commit
cdae08f3c3
2 changed files with 124 additions and 9 deletions
|
@ -1,3 +1,6 @@
|
||||||
|
import Tabs from '@theme/Tabs';
|
||||||
|
import TabItem from '@theme/TabItem';
|
||||||
|
|
||||||
# OpenAI
|
# OpenAI
|
||||||
LiteLLM supports OpenAI Chat + Text completion and embedding calls.
|
LiteLLM supports OpenAI Chat + Text completion and embedding calls.
|
||||||
|
|
||||||
|
@ -22,6 +25,113 @@ response = completion(
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Proxy Usage
|
||||||
|
|
||||||
|
Here's how to call OpenAI models with the LiteLLM Proxy Server
|
||||||
|
|
||||||
|
### 1. Save key in your environment
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OPENAI_API_KEY=""
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Start the proxy
|
||||||
|
|
||||||
|
<Tabs>
|
||||||
|
<TabItem value="cli" label="CLI">
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ litellm --model gpt-3.5-turbo
|
||||||
|
|
||||||
|
# Server running on http://0.0.0.0:4000
|
||||||
|
```
|
||||||
|
</TabItem>
|
||||||
|
<TabItem value="config" label="config.yaml">
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
model_list:
|
||||||
|
- model_name: gpt-3.5-turbo
|
||||||
|
litellm_params:
|
||||||
|
model: gpt-3.5-turbo
|
||||||
|
```
|
||||||
|
</TabItem>
|
||||||
|
</Tabs>
|
||||||
|
|
||||||
|
### 3. Test it
|
||||||
|
|
||||||
|
|
||||||
|
<Tabs>
|
||||||
|
<TabItem value="Curl" label="Curl Request">
|
||||||
|
|
||||||
|
```shell
|
||||||
|
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||||
|
--header 'Content-Type: application/json' \
|
||||||
|
--data ' {
|
||||||
|
"model": "gpt-3.5-turbo",
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "what llm are you"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
'
|
||||||
|
```
|
||||||
|
</TabItem>
|
||||||
|
<TabItem value="openai" label="OpenAI v1.0.0+">
|
||||||
|
|
||||||
|
```python
|
||||||
|
import openai
|
||||||
|
client = openai.OpenAI(
|
||||||
|
api_key="anything",
|
||||||
|
base_url="http://0.0.0.0:4000"
|
||||||
|
)
|
||||||
|
|
||||||
|
# request sent to model set on litellm proxy, `litellm --model`
|
||||||
|
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "this is a test request, write a short poem"
|
||||||
|
}
|
||||||
|
])
|
||||||
|
|
||||||
|
print(response)
|
||||||
|
|
||||||
|
```
|
||||||
|
</TabItem>
|
||||||
|
<TabItem value="langchain" label="Langchain">
|
||||||
|
|
||||||
|
```python
|
||||||
|
from langchain.chat_models import ChatOpenAI
|
||||||
|
from langchain.prompts.chat import (
|
||||||
|
ChatPromptTemplate,
|
||||||
|
HumanMessagePromptTemplate,
|
||||||
|
SystemMessagePromptTemplate,
|
||||||
|
)
|
||||||
|
from langchain.schema import HumanMessage, SystemMessage
|
||||||
|
|
||||||
|
chat = ChatOpenAI(
|
||||||
|
openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy
|
||||||
|
model = "gpt-3.5-turbo",
|
||||||
|
temperature=0.1
|
||||||
|
)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
SystemMessage(
|
||||||
|
content="You are a helpful assistant that im using to make a test request to."
|
||||||
|
),
|
||||||
|
HumanMessage(
|
||||||
|
content="test from litellm. tell me why it's amazing in 1 sentence"
|
||||||
|
),
|
||||||
|
]
|
||||||
|
response = chat(messages)
|
||||||
|
|
||||||
|
print(response)
|
||||||
|
```
|
||||||
|
</TabItem>
|
||||||
|
</Tabs>
|
||||||
|
|
||||||
|
|
||||||
### Optional Keys - OpenAI Organization, OpenAI API Base
|
### Optional Keys - OpenAI Organization, OpenAI API Base
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
|
|
@ -4,18 +4,23 @@ model_list:
|
||||||
model: openai/my-fake-model
|
model: openai/my-fake-model
|
||||||
api_key: my-fake-key
|
api_key: my-fake-key
|
||||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||||
|
- model_name: gpt-instruct
|
||||||
|
litellm_params:
|
||||||
|
model: gpt-3.5-turbo-instruct
|
||||||
|
# api_key: my-fake-key
|
||||||
|
# api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||||
|
|
||||||
litellm_settings:
|
# litellm_settings:
|
||||||
cache: true
|
# cache: true
|
||||||
# max_budget: 600020
|
# max_budget: 600020
|
||||||
# budget_duration: 30d
|
# budget_duration: 30d
|
||||||
|
|
||||||
general_settings:
|
general_settings:
|
||||||
master_key: sk-1234
|
master_key: sk-1234
|
||||||
proxy_batch_write_at: 60 # 👈 Frequency of batch writing logs to server (in seconds)
|
# proxy_batch_write_at: 60 # 👈 Frequency of batch writing logs to server (in seconds)
|
||||||
enable_jwt_auth: True
|
# enable_jwt_auth: True
|
||||||
alerting: ["slack"]
|
# alerting: ["slack"]
|
||||||
litellm_jwtauth:
|
# litellm_jwtauth:
|
||||||
admin_jwt_scope: "litellm_proxy_admin"
|
# admin_jwt_scope: "litellm_proxy_admin"
|
||||||
team_jwt_scope: "litellm_team"
|
# team_jwt_scope: "litellm_team"
|
||||||
public_key_ttl: 600
|
# public_key_ttl: 600
|
Loading…
Add table
Add a link
Reference in a new issue