From f7945a945b24678f64f9d6dbb59d3c8e581353e0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Mar 2024 21:07:55 -0700 Subject: [PATCH 1/2] (docs) add example using vertex ai on litellm proxy --- docs/my-website/docs/providers/vertex.md | 115 ++++++++++++++++------- 1 file changed, 81 insertions(+), 34 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index fa18525ee..05720db13 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -23,58 +23,105 @@ litellm.vertex_location = "us-central1" # proj location response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]) ``` -## OpenAI Proxy Usage +## Usage with LiteLLM Proxy Server Here's how to use Vertex AI with the LiteLLM Proxy Server 1. Modify the config.yaml - + - + -Use this when you need to set a different location for each vertex model + Use this when you need to set a different location for each vertex model -```yaml -model_list: - - model_name: gemini-vision - litellm_params: - model: vertex_ai/gemini-1.0-pro-vision-001 - vertex_project: "project-id" - vertex_location: "us-central1" - - model_name: gemini-vision - litellm_params: - model: vertex_ai/gemini-1.0-pro-vision-001 - vertex_project: "project-id2" - vertex_location: "us-east" -``` + ```yaml + model_list: + - model_name: gemini-vision + litellm_params: + model: vertex_ai/gemini-1.0-pro-vision-001 + vertex_project: "project-id" + vertex_location: "us-central1" + - model_name: gemini-vision + litellm_params: + model: vertex_ai/gemini-1.0-pro-vision-001 + vertex_project: "project-id2" + vertex_location: "us-east" + ``` - + - + -Use this when you have one vertex location for all models + Use this when you have one vertex location for all models -```yaml -litellm_settings: - vertex_project: "hardy-device-38811" # Your Project ID - vertex_location: "us-central1" # proj location + ```yaml + litellm_settings: + vertex_project: "hardy-device-38811" # Your Project ID + vertex_location: "us-central1" # proj location -model_list: - -model_name: team1-gemini-pro - litellm_params: - model: gemini-pro -``` + model_list: + -model_name: team1-gemini-pro + litellm_params: + model: gemini-pro + ``` - + - + 2. Start the proxy -```bash -$ litellm --config /path/to/config.yaml -``` + ```bash + $ litellm --config /path/to/config.yaml + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="team1-gemini-pro", + messages = [ + { + "role": "user", + "content": "what llm are you" + } + ], + ) + + print(response) + ``` + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "team1-gemini-pro", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' + ``` + + + ## Set Vertex Project & Vertex Location All calls using Vertex AI require the following parameters: From ee1a0a940902f20c5fc108c4f9b293b84c6354ca Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Mar 2024 21:15:50 -0700 Subject: [PATCH 2/2] (docs) add example using openai compatib model --- .../docs/providers/openai_compatible.md | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/openai_compatible.md b/docs/my-website/docs/providers/openai_compatible.md index f86544c28..09dcd7e4c 100644 --- a/docs/my-website/docs/providers/openai_compatible.md +++ b/docs/my-website/docs/providers/openai_compatible.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # OpenAI-Compatible Endpoints To call models hosted behind an openai proxy, make 2 changes: @@ -39,4 +42,74 @@ response = litellm.embedding( input=["good morning from litellm"] ) print(response) -``` \ No newline at end of file +``` + + + +## Usage with LiteLLM Proxy Server + +Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server + +1. Modify the config.yaml + + ```yaml + model_list: + - model_name: my-model + litellm_params: + model: openai/ # add openai/ prefix to route as OpenAI provider + api_base: # add api base for OpenAI compatible provider + api_key: api-key # api key to send your model + ``` + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="my-model", + messages = [ + { + "role": "user", + "content": "what llm are you" + } + ], + ) + + print(response) + ``` + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "my-model", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' + ``` + + +