diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 3d0f81e87..ab35abfa7 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -48,10 +48,10 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea |Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | |Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | -|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ | | | | | +|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ | | | | |AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |VertexAI| ✅ | ✅ | | ✅ | ✅ | | | | | | | | | ✅ | ✅ | | | -|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (for anthropic) | | +|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (model dependent) | | |Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | |AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | @@ -61,6 +61,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea |Ollama| ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | ✅ | | |✅| | | | | | | |Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | | |ClarifAI| ✅ | ✅ | |✅ | ✅ | | | | | | | | | | | +|Github| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ (model dependent)|✅ (model dependent)| | | :::note By default, LiteLLM raises an exception if the openai param being passed in isn't supported. diff --git a/docs/my-website/docs/providers/github.md b/docs/my-website/docs/providers/github.md new file mode 100644 index 000000000..1ec7060a1 --- /dev/null +++ b/docs/my-website/docs/providers/github.md @@ -0,0 +1,261 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 🆕 Github +https://github.com/marketplace/models + +:::tip + +**We support ALL Github models, just set `model=groq/` as a prefix when sending litellm requests** + +::: + +## API Key +```python +# env variable +os.environ['GITHUB_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['GITHUB_API_KEY'] = "" +response = completion( + model="github/llama3-8b-8192", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['GITHUB_API_KEY'] = "" +response = completion( + model="github/llama3-8b-8192", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + + + +## Usage with LiteLLM Proxy + +### 1. Set Github Models on config.yaml + +```yaml +model_list: + - model_name: github-llama3-8b-8192 # Model Alias to use for requests + litellm_params: + model: github/llama3-8b-8192 + api_key: "os.environ/GITHUB_API_KEY" # ensure you have `GITHUB_API_KEY` in your .env +``` + +### 2. Start Proxy + +``` +litellm --config config.yaml +``` + +### 3. Test it + +Make request to litellm proxy + + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "github-llama3-8b-8192", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create(model="github-llama3-8b-8192", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +]) + +print(response) + +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy + model = "github-llama3-8b-8192", + temperature=0.1 +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + +## Supported Models - ALL Github Models Supported! +We support ALL Github models, just set `github/` as a prefix when sending completion requests + +| Model Name | Usage | +|--------------------|---------------------------------------------------------| +| llama-3.1-8b-instant | `completion(model="github/llama-3.1-8b-instant", messages)` | +| llama-3.1-70b-versatile | `completion(model="github/llama-3.1-70b-versatile", messages)` | +| llama-3.1-405b-reasoning | `completion(model="github/llama-3.1-405b-reasoning", messages)` | +| llama3-8b-8192 | `completion(model="github/llama3-8b-8192", messages)` | +| llama3-70b-8192 | `completion(model="github/llama3-70b-8192", messages)` | +| llama2-70b-4096 | `completion(model="github/llama2-70b-4096", messages)` | +| mixtral-8x7b-32768 | `completion(model="github/mixtral-8x7b-32768", messages)` | +| gemma-7b-it | `completion(model="github/gemma-7b-it", messages)` | + +## Github - Tool / Function Calling Example + +```python +# Example dummy function hard coded to return the current weather +import json +def get_current_weather(location, unit="fahrenheit"): + """Get the current weather in a given location""" + if "tokyo" in location.lower(): + return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) + elif "san francisco" in location.lower(): + return json.dumps( + {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} + ) + elif "paris" in location.lower(): + return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) + + + + +# Step 1: send the conversation and available functions to the model +messages = [ + { + "role": "system", + "content": "You are a function calling LLM that uses the data extracted from get_current_weather to answer questions about the weather in San Francisco.", + }, + { + "role": "user", + "content": "What's the weather like in San Francisco?", + }, +] +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } +] +response = litellm.completion( + model="github/llama3-8b-8192", + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit +) +print("Response\n", response) +response_message = response.choices[0].message +tool_calls = response_message.tool_calls + + +# Step 2: check if the model wanted to call a function +if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + } + messages.append( + response_message + ) # extend conversation with assistant's reply + print("Response message\n", response_message) + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + function_to_call = available_functions[function_name] + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + print(f"messages: {messages}") + second_response = litellm.completion( + model="github/llama3-8b-8192", messages=messages + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 61aca1169..69fd32cb3 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -161,6 +161,7 @@ const sidebars = { "providers/perplexity", "providers/friendliai", "providers/groq", + "providers/github", "providers/deepseek", "providers/fireworks_ai", "providers/clarifai",