From fe0b0ddaaa676272e7d6d8bc53f91fd7af9543f4 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Wed, 24 Jul 2024 14:33:49 -0700
Subject: [PATCH] doc example using litellm proxy with groq

---
 docs/my-website/docs/providers/groq.md | 106 ++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 4 deletions(-)

diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md
index bcca20b5d..bfb944cb4 100644
--- a/docs/my-website/docs/providers/groq.md
+++ b/docs/my-website/docs/providers/groq.md
@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Groq
 https://groq.com/
 
@@ -20,7 +23,7 @@ import os
 
 os.environ['GROQ_API_KEY'] = ""
 response = completion(
-    model="groq/llama2-70b-4096", 
+    model="groq/llama3-8b-8192", 
     messages=[
        {"role": "user", "content": "hello from litellm"}
    ],
@@ -35,7 +38,7 @@ import os
 
 os.environ['GROQ_API_KEY'] = ""
 response = completion(
-    model="groq/llama2-70b-4096", 
+    model="groq/llama3-8b-8192", 
     messages=[
        {"role": "user", "content": "hello from litellm"}
    ],
@@ -47,6 +50,101 @@ for chunk in response:
 ```
 
 
+
+## Usage with LiteLLM Proxy 
+
+### 1. Set Groq Models on config.yaml
+
+```yaml
+model_list:
+  - model_name: groq-llama3-8b-8192 # Model Alias to use for requests
+    litellm_params:
+      model: groq/llama3-8b-8192
+      api_key: "os.environ/GROQ_API_KEY" # ensure you have `GROQ_API_KEY` in your .env
+```
+
+### 2. Start Proxy 
+
+```
+litellm --config config.yaml
+```
+
+### 3. Test it
+
+Make request to litellm proxy
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data ' {
+      "model": "groq-llama3-8b-8192",
+      "messages": [
+        {
+          "role": "user",
+          "content": "what llm are you"
+        }
+      ]
+    }
+'
+```
+</TabItem>
+<TabItem value="openai" label="OpenAI v1.0.0+">
+
+```python
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(model="groq-llama3-8b-8192", messages = [
+    {
+        "role": "user",
+        "content": "this is a test request, write a short poem"
+    }
+])
+
+print(response)
+
+```
+</TabItem>
+<TabItem value="langchain" label="Langchain">
+
+```python
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+    SystemMessagePromptTemplate,
+)
+from langchain.schema import HumanMessage, SystemMessage
+
+chat = ChatOpenAI(
+    openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy
+    model = "groq-llama3-8b-8192",
+    temperature=0.1
+)
+
+messages = [
+    SystemMessage(
+        content="You are a helpful assistant that im using to make a test request to."
+    ),
+    HumanMessage(
+        content="test from litellm. tell me why it's amazing in 1 sentence"
+    ),
+]
+response = chat(messages)
+
+print(response)
+```
+</TabItem>
+</Tabs>
+
+
+
 ## Supported Models - ALL Groq Models Supported!
 We support ALL Groq models, just set `groq/` as a prefix when sending completion requests
 
@@ -114,7 +212,7 @@ tools = [
     }
 ]
 response = litellm.completion(
-    model="groq/llama2-70b-4096",
+    model="groq/llama3-8b-8192",
     messages=messages,
     tools=tools,
     tool_choice="auto",  # auto is default, but we'll be explicit
@@ -154,7 +252,7 @@ if tool_calls:
         )  # extend conversation with function response
     print(f"messages: {messages}")
     second_response = litellm.completion(
-        model="groq/llama2-70b-4096", messages=messages
+        model="groq/llama3-8b-8192", messages=messages
     )  # get a new response from the model where it can see the function response
     print("second response\n", second_response)
 ```