From 841ac5ab20798b8e451774178c0c328f0e6ebd29 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 28 Aug 2023 06:53:27 -0700 Subject: [PATCH 1/8] adding support for openai api base --- .env.example | 1 + docs/my-website/docs/completion/supported.md | 2 ++ litellm/main.py | 6 ++---- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 36eff8a43..d99c05d37 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # OpenAI OPENAI_API_KEY = "" +OPENAI_API_BASE = "" # Cohere COHERE_API_KEY = "" # OpenRouter diff --git a/docs/my-website/docs/completion/supported.md b/docs/my-website/docs/completion/supported.md index e3719938c..9988b08e7 100644 --- a/docs/my-website/docs/completion/supported.md +++ b/docs/my-website/docs/completion/supported.md @@ -17,6 +17,8 @@ liteLLM reads key naming, all keys should be named in the following format: | gpt-3.5-turbo-16k-0613 | `completion('gpt-3.5-turbo-16k-0613', messages)` | `os.environ['OPENAI_API_KEY']` | | gpt-4 | `completion('gpt-4', messages)` | `os.environ['OPENAI_API_KEY']` | +These also support the `OPENAI_API_BASE` environment variable, which can be used to specify a custom API endpoint. + ### Azure OpenAI Chat Completion Models | Model Name | Function Call | Required OS Variables | diff --git a/litellm/main.py b/litellm/main.py index 6a48c7f57..6e26bf3e5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -211,11 +211,9 @@ def completion( openai.api_type = "openai" # note: if a user sets a custom base - we should ensure this works api_base = ( - custom_api_base if custom_api_base is not None else litellm.api_base - ) # allow for the setting of dynamic and stateful api-bases - openai.api_base = ( - api_base if api_base is not None else "https://api.openai.com/v1" + custom_api_base or litellm.api_base or get_secret("OPENAI_API_BASE") ) + openai.api_base = api_base or "https://api.openai.com/v1" openai.api_version = None if litellm.organization: openai.organization = litellm.organization From fe4184962c6eb7fc5d13d131f3b0bda1f90e28c4 Mon Sep 17 00:00:00 2001 From: Gal Kleinman Date: Mon, 28 Aug 2023 17:05:21 +0300 Subject: [PATCH 2/8] fix: context propagation on acompletion method --- litellm/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 6e26bf3e5..001a6c546 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1,7 +1,7 @@ import os, openai, sys from typing import Any from functools import partial -import dotenv, traceback, random, asyncio, time +import dotenv, traceback, random, asyncio, time, contextvars from copy import deepcopy import litellm from litellm import ( # type: ignore @@ -49,8 +49,12 @@ async def acompletion(*args, **kwargs): # Use a partial function to pass your keyword arguments func = partial(completion, *args, **kwargs) + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + # Call the synchronous function using run_in_executor - return await loop.run_in_executor(None, func) + return await loop.run_in_executor(None, func_with_context) @client From 9497175a97ec396282f97c50bbf4aecdad540b32 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 28 Aug 2023 07:21:02 -0700 Subject: [PATCH 3/8] update get secret logic --- litellm/main.py | 4 ++-- litellm/utils.py | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 6e26bf3e5..a6d94b517 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -211,9 +211,9 @@ def completion( openai.api_type = "openai" # note: if a user sets a custom base - we should ensure this works api_base = ( - custom_api_base or litellm.api_base or get_secret("OPENAI_API_BASE") + custom_api_base or litellm.api_base or get_secret("OPENAI_API_BASE") or "https://api.openai.com/v1" ) - openai.api_base = api_base or "https://api.openai.com/v1" + openai.api_base = api_base openai.api_version = None if litellm.organization: openai.organization = litellm.organization diff --git a/litellm/utils.py b/litellm/utils.py index 309d90396..eac0079cf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1444,13 +1444,12 @@ def get_secret(secret_name): if litellm.secret_manager_client != None: # TODO: check which secret manager is being used # currently only supports Infisical - secret = litellm.secret_manager_client.get_secret( - secret_name).secret_value - if secret != None: - return secret # if secret found in secret manager return it - else: - raise ValueError( - f"Secret '{secret_name}' not found in secret manager") + try: + secret = litellm.secret_manager_client.get_secret( + secret_name).secret_value + except: + secret = None + return secret elif litellm.api_key != None: # if users use litellm default key return litellm.api_key else: From 3e14262637bb6bdab61bd0eb71fcf0d534eaf19c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 28 Aug 2023 07:29:48 -0700 Subject: [PATCH 4/8] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c4de3c368..432e65343 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.492" +version = "0.1.493" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License" From dfdf47ddd06318ca7c7609ea0905d5cd92fb7d1b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 28 Aug 2023 08:04:05 -0700 Subject: [PATCH 5/8] add text completion format support --- .../docs/tutorials/text_completion.md | 38 +++++++++++++++++++ litellm/main.py | 5 ++- litellm/tests/test_completion.py | 11 ++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/tutorials/text_completion.md diff --git a/docs/my-website/docs/tutorials/text_completion.md b/docs/my-website/docs/tutorials/text_completion.md new file mode 100644 index 000000000..3eb95592a --- /dev/null +++ b/docs/my-website/docs/tutorials/text_completion.md @@ -0,0 +1,38 @@ +# Using Text Completion Format - with Completion() + +If your prefer interfacing with the OpenAI Text Completion format this tutorial covers how to use LiteLLM in this format +```python +response = openai.Completion.create( + model="text-davinci-003", + prompt='Write a tagline for a traditional bavarian tavern', + temperature=0, + max_tokens=100) +``` + +## Using LiteLLM in the Text Completion format +### With gpt-3.5-turbo +```python +response = openai.Completion.create( + model="gpt-3.5-turbo", + prompt='Write a tagline for a traditional bavarian tavern', + temperature=0, + max_tokens=100) +``` + +### With text-davinci-003 +```python +response = openai.Completion.create( + model="text-davinci-003", + prompt='Write a tagline for a traditional bavarian tavern', + temperature=0, + max_tokens=100) +``` + +### With llama2 +```python +response = openai.Completion.create( + model="togethercomputer/llama-2-70b-chat", + prompt='Write a tagline for a traditional bavarian tavern', + temperature=0, + max_tokens=100) +``` \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index a327a7420..a161e663b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -64,8 +64,8 @@ async def acompletion(*args, **kwargs): ) ## set timeouts, in case calls hang (e.g. Azure) - default is 600s, override with `force_timeout` def completion( model, - messages, # required params # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create + messages=[], functions=[], function_call="", # optional params temperature=1, @@ -91,6 +91,7 @@ def completion( custom_llm_provider=None, custom_api_base=None, litellm_call_id=None, + prompt="", # allow completion to be used as textCompletion or as ChatCompletion # model specific optional params # used by text-bison only top_k=40, @@ -101,6 +102,8 @@ def completion( try: if fallbacks != []: return completion_with_fallbacks(**args) + if messages == [] and prompt!="": + messages = [{"role": "user", "content": prompt}] if litellm.model_alias_map and model in litellm.model_alias_map: args["model_alias_map"] = litellm.model_alias_map model = litellm.model_alias_map[model] # update the model to the actual value if an alias has been passed in diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 1bd36790b..6aa7612d6 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -142,6 +142,17 @@ def test_completion_openai(): except Exception as e: pytest.fail(f"Error occurred: {e}") +def test_completion_openai_prompt(): + try: + response = completion(model="gpt-3.5-turbo", prompt="What's the weather in SF?") + response_str = response["choices"][0]["message"]["content"] + response_str_2 = response.choices[0].message.content + print(response) + assert response_str == response_str_2 + assert type(response_str) == str + assert len(response_str) > 1 + except Exception as e: + pytest.fail(f"Error occurred: {e}") def test_completion_text_openai(): try: From 0dd9231e2cad8865550bd9d13492fc7923448c87 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 28 Aug 2023 08:07:51 -0700 Subject: [PATCH 6/8] docs update --- docs/my-website/sidebars.js | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index cfb24121a..05d80b0b1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -43,6 +43,7 @@ const sidebars = { 'tutorials/TogetherAI_liteLLM', 'tutorials/fallbacks', 'tutorials/finetuned_chat_gpt', + 'tutorials/text_completion', 'tutorials/ab_test_llms', 'tutorials/litellm_Test_Multiple_Providers' ], From fedd16361f270d9b84a3bf6f9c2d7f14475fce98 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 28 Aug 2023 08:20:20 -0700 Subject: [PATCH 7/8] add text_completion --- docs/my-website/docs/tutorials/text_completion.md | 7 ++++--- litellm/main.py | 10 +++++++--- litellm/tests/test_completion.py | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/my-website/docs/tutorials/text_completion.md b/docs/my-website/docs/tutorials/text_completion.md index 3eb95592a..1d210076e 100644 --- a/docs/my-website/docs/tutorials/text_completion.md +++ b/docs/my-website/docs/tutorials/text_completion.md @@ -12,7 +12,8 @@ response = openai.Completion.create( ## Using LiteLLM in the Text Completion format ### With gpt-3.5-turbo ```python -response = openai.Completion.create( +from litellm import text_completion +response = text_completion( model="gpt-3.5-turbo", prompt='Write a tagline for a traditional bavarian tavern', temperature=0, @@ -21,7 +22,7 @@ response = openai.Completion.create( ### With text-davinci-003 ```python -response = openai.Completion.create( +response = text_completion( model="text-davinci-003", prompt='Write a tagline for a traditional bavarian tavern', temperature=0, @@ -30,7 +31,7 @@ response = openai.Completion.create( ### With llama2 ```python -response = openai.Completion.create( +response = text_completion( model="togethercomputer/llama-2-70b-chat", prompt='Write a tagline for a traditional bavarian tavern', temperature=0, diff --git a/litellm/main.py b/litellm/main.py index a161e663b..4de27dee1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -91,7 +91,6 @@ def completion( custom_llm_provider=None, custom_api_base=None, litellm_call_id=None, - prompt="", # allow completion to be used as textCompletion or as ChatCompletion # model specific optional params # used by text-bison only top_k=40, @@ -102,8 +101,6 @@ def completion( try: if fallbacks != []: return completion_with_fallbacks(**args) - if messages == [] and prompt!="": - messages = [{"role": "user", "content": prompt}] if litellm.model_alias_map and model in litellm.model_alias_map: args["model_alias_map"] = litellm.model_alias_map model = litellm.model_alias_map[model] # update the model to the actual value if an alias has been passed in @@ -867,6 +864,13 @@ def embedding( custom_llm_provider="azure" if azure == True else None, ) +###### Text Completion ################ +def text_completion(*args, **kwargs): + if 'prompt' in kwargs: + messages = [{'role': 'system', 'content': kwargs['prompt']}] + kwargs['messages'] = messages + kwargs.pop('prompt') + return completion(*args, **kwargs) ####### HELPER FUNCTIONS ################ ## Set verbose to true -> ```litellm.set_verbose = True``` diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 6aa7612d6..e59448450 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -10,7 +10,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest import litellm -from litellm import embedding, completion +from litellm import embedding, completion, text_completion # from infisical import InfisicalClient @@ -144,7 +144,7 @@ def test_completion_openai(): def test_completion_openai_prompt(): try: - response = completion(model="gpt-3.5-turbo", prompt="What's the weather in SF?") + response = text_completion(model="gpt-3.5-turbo", prompt="What's the weather in SF?") response_str = response["choices"][0]["message"]["content"] response_str_2 = response.choices[0].message.content print(response) From de101229bbcc72e1d41f3c596dece00689f8d640 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 28 Aug 2023 08:24:54 -0700 Subject: [PATCH 8/8] bump v --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 432e65343..d787fa222 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.493" +version = "0.1.494" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"