phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Update boto3 dependency to version 1.28.57, refactor bedrock client initialization and remove troubleshooting guide from documentation.

Browse source
This commit is contained in:
coconut49 2023-09-30 12:31:26 +08:00
parent b8c8f01221
commit 4f67261c3c
No known key found for this signature in database
9 changed files with 186 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.circleci/config.yml
View file

@ -32,8 +32,7 @@ jobs:
pip install -q google-generativeai
pip install openai[datalib]
pip install -Uq chromadb==0.3.29
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl
pip install "boto3>=1.28.57"
- save_cache:
paths:
- ./venv

BIN
cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl
View file

Binary file not shown.

BIN
cookbook/bedrock_resources/boto3-1.28.21.tar.gz
View file

Binary file not shown.

BIN
cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl
View file

Binary file not shown.

BIN
cookbook/bedrock_resources/botocore-1.31.21.tar.gz
View file

Binary file not shown.

10
docs/my-website/docs/providers/bedrock.md
View file

@ -75,13 +75,3 @@ for chunk in response:
}
}
```
## Troubleshooting
If creating a boto3 bedrock client fails with `Unknown service: 'bedrock'`
Try re installing boto3 using the following commands
```shell
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl
```
See Page 26 on [Amazon Bedrock User Guide](https://d2eo22ngex1n9g.cloudfront.net/Documentation/BedrockUserGuide.pdf)

113
litellm/llms/bedrock.py
View file

@ -1,11 +1,8 @@
import os
import json
from enum import Enum
import requests
import time
from typing import Callable
from litellm.utils import ModelResponse, get_secret
import sys
class BedrockError(Exception):
def __init__(self, status_code, message):
@ -15,38 +12,24 @@ class BedrockError(Exception):
self.message
) # Call the base class constructor with the parameters it needs
class AnthropicConstants(Enum):
HUMAN_PROMPT = "\n\nHuman:"
AI_PROMPT = "\n\nAssistant:"
def init_bedrock_client(region_name):
import sys
import boto3
import subprocess
try:
client = boto3.client(
service_name="bedrock",
region_name=region_name,
endpoint_url=f'https://bedrock.{region_name}.amazonaws.com'
)
except Exception as e:
try:
command1 = "python3 -m pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl"
subprocess.run(command1, shell=True, check=True)
# Command 2: Install boto3 from URL
command2 = "python3 -m pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl"
subprocess.run(command2, shell=True, check=True)
import boto3
client = boto3.client(
service_name="bedrock",
region_name=region_name,
endpoint_url=f'https://bedrock.{region_name}.amazonaws.com'
)
except Exception as e:
raise e
def init_bedrock_client(region_name):
import boto3
client = boto3.client(
service_name="bedrock-runtime",
region_name=region_name,
endpoint_url=f'https://bedrock-runtime.{region_name}.amazonaws.com'
)
return client
def convert_messages_to_prompt(messages, provider):
# handle anthropic prompts using anthropic constants
if provider == "anthropic":
@ -80,30 +63,31 @@ def convert_messages_to_prompt(messages, provider):
prompt += f"{message['content']}"
return prompt
"""
BEDROCK AUTH Keys/Vars
os.environ['AWS_ACCESS_KEY_ID'] = ""
os.environ['AWS_SECRET_ACCESS_KEY'] = ""
"""
# set os.environ['AWS_REGION_NAME'] = <your-region_name>
def completion(
model: str,
messages: list,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj,
optional_params=None,
stream=False,
litellm_params=None,
logger_fn=None,
model: str,
messages: list,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj,
optional_params=None,
stream=False,
litellm_params=None,
logger_fn=None,
):
region_name = (
get_secret("AWS_REGION_NAME") or
"us-west-2" # default to us-west-2 if user not specified
get_secret("AWS_REGION_NAME") or
"us-west-2" # default to us-west-2 if user not specified
)
client = init_bedrock_client(region_name)
@ -119,48 +103,48 @@ def completion(
elif provider == "ai21":
data = json.dumps({
"prompt": prompt,
})
})
else: # amazon titan
else: # amazon titan
data = json.dumps({
"inputText": prompt,
"inputText": prompt,
"textGenerationConfig": optional_params,
})
## LOGGING
})
## LOGGING
logging_obj.pre_call(
input=prompt,
api_key="",
additional_args={"complete_input_dict": data},
)
input=prompt,
api_key="",
additional_args={"complete_input_dict": data},
)
## COMPLETION CALL
accept = 'application/json'
contentType = 'application/json'
if stream == True:
response = client.invoke_model_with_response_stream(
body=data,
modelId=model,
accept=accept,
body=data,
modelId=model,
accept=accept,
contentType=contentType
)
response = response.get('body')
return response
response = client.invoke_model(
body=data,
modelId=model,
accept=accept,
body=data,
modelId=model,
accept=accept,
contentType=contentType
)
response_body = json.loads(response.get('body').read())
## LOGGING
logging_obj.post_call(
input=prompt,
api_key="",
original_response=response,
additional_args={"complete_input_dict": data},
)
input=prompt,
api_key="",
original_response=response,
additional_args={"complete_input_dict": data},
)
print_verbose(f"raw model_response: {response}")
## RESPONSE OBJECT
outputText = "default"
@ -169,7 +153,7 @@ def completion(
elif provider == "anthropic":
outputText = response_body['completion']
model_response["finish_reason"] = response_body["stop_reason"]
else: # amazon titan
else: # amazon titan
outputText = response_body.get('results')[0].get('outputText')
if "error" in outputText:
raise BedrockError(
@ -185,7 +169,7 @@ def completion(
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
prompt_tokens = len(
encoding.encode(prompt)
)
)
completion_tokens = len(
encoding.encode(model_response["choices"][0]["message"]["content"])
)
@ -199,6 +183,7 @@ def completion(
}
return model_response
def embedding():
# logic for parsing in - calling - parsing out model embedding calls
pass

146
poetry.lock generated
View file

@ -1,9 +1,10 @@
# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
[[package]]
name = "aiohttp"
version = "3.8.5"
description = "Async http client/server framework (asyncio)"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -112,6 +113,7 @@ speedups = ["Brotli", "aiodns", "cchardet"]
name = "aiosignal"
version = "1.3.1"
description = "aiosignal: a list of registered asynchronous callbacks"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -126,6 +128,7 @@ frozenlist = ">=1.1.0"
name = "async-timeout"
version = "4.0.3"
description = "Timeout context manager for asyncio programs"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -137,6 +140,7 @@ files = [
name = "attrs"
version = "23.1.0"
description = "Classes Without Boilerplate"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -151,10 +155,51 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-
tests = ["attrs[tests-no-zope]", "zope-interface"]
tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
[[package]]
name = "boto3"
version = "1.28.57"
description = "The AWS SDK for Python"
category = "main"
optional = false
python-versions = ">= 3.7"
files = [
{file = "boto3-1.28.57-py3-none-any.whl", hash = "sha256:5ddf24cf52c7fb6aaa332eaa08ae8c2afc8f2d1e8860680728533dd573904e32"},
{file = "boto3-1.28.57.tar.gz", hash = "sha256:e2d2824ba6459b330d097e94039a9c4f96ae3f4bcdc731d620589ad79dcd16d3"},
]
[package.dependencies]
botocore = ">=1.31.57,<1.32.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.7.0,<0.8.0"
[package.extras]
crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.31.57"
description = "Low-level, data-driven core of boto 3."
category = "main"
optional = false
python-versions = ">= 3.7"
files = [
{file = "botocore-1.31.57-py3-none-any.whl", hash = "sha256:af006248276ff8e19e3ec7214478f6257035eb40aed865e405486500471ae71b"},
{file = "botocore-1.31.57.tar.gz", hash = "sha256:301436174635bec739b225b840fc365ca00e5c1a63e5b2a19ee679d204e01b78"},
]
[package.dependencies]
jmespath = ">=0.7.1,<2.0.0"
python-dateutil = ">=2.1,<3.0.0"
urllib3 = ">=1.25.4,<1.27"
[package.extras]
crt = ["awscrt (==0.16.26)"]
[[package]]
name = "certifi"
version = "2023.7.22"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -166,6 +211,7 @@ files = [
name = "charset-normalizer"
version = "3.2.0"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
python-versions = ">=3.7.0"
files = [
@ -250,6 +296,7 @@ files = [
name = "click"
version = "8.1.7"
description = "Composable command line interface toolkit"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -264,6 +311,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [
@ -275,6 +323,7 @@ files = [
name = "filelock"
version = "3.12.4"
description = "A platform independent file lock."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -291,6 +340,7 @@ typing = ["typing-extensions (>=4.7.1)"]
name = "frozenlist"
version = "1.4.0"
description = "A list-like structure which implements collections.abc.MutableSequence"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -361,6 +411,7 @@ files = [
name = "fsspec"
version = "2023.9.2"
description = "File-system specification"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -396,6 +447,7 @@ tqdm = ["tqdm"]
name = "huggingface-hub"
version = "0.16.4"
description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
category = "main"
optional = false
python-versions = ">=3.7.0"
files = [
@ -428,6 +480,7 @@ typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "typ
name = "idna"
version = "3.4"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.5"
files = [
@ -439,6 +492,7 @@ files = [
name = "importlib-metadata"
version = "6.8.0"
description = "Read metadata from Python packages"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -454,10 +508,23 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker
perf = ["ipython"]
testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"]
[[package]]
name = "jmespath"
version = "1.0.1"
description = "JSON Matching Expressions"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
{file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
]
[[package]]
name = "multidict"
version = "6.0.4"
description = "multidict implementation"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -541,6 +608,7 @@ files = [
name = "openai"
version = "0.28.1"
description = "Python client library for the OpenAI API"
category = "main"
optional = false
python-versions = ">=3.7.1"
files = [
@ -555,7 +623,7 @@ tqdm = "*"
[package.extras]
datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"]
dev = ["black (>=21.6b0,<22.0)", "pytest (>=6.0.0,<7.0.0)", "pytest-asyncio", "pytest-mock"]
embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"]
wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"]
@ -563,6 +631,7 @@ wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1
name = "packaging"
version = "23.1"
description = "Core utilities for Python packages"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -570,10 +639,26 @@ files = [
{file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"},
]
[[package]]
name = "python-dateutil"
version = "2.8.2"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
files = [
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
{file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
]
[package.dependencies]
six = ">=1.5"
[[package]]
name = "python-dotenv"
version = "1.0.0"
description = "Read key-value pairs from a .env file and set them as environment variables"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -588,6 +673,7 @@ cli = ["click (>=5.0)"]
name = "pyyaml"
version = "6.0.1"
description = "YAML parser and emitter for Python"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -647,6 +733,7 @@ files = [
name = "regex"
version = "2023.8.8"
description = "Alternative regular expression module, to replace re."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -744,6 +831,7 @@ files = [
name = "requests"
version = "2.31.0"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -761,10 +849,41 @@ urllib3 = ">=1.21.1,<3"
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
[[package]]
name = "s3transfer"
version = "0.7.0"
description = "An Amazon S3 Transfer Manager"
category = "main"
optional = false
python-versions = ">= 3.7"
files = [
{file = "s3transfer-0.7.0-py3-none-any.whl", hash = "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a"},
{file = "s3transfer-0.7.0.tar.gz", hash = "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e"},
]
[package.dependencies]
botocore = ">=1.12.36,<2.0a.0"
[package.extras]
crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"]
[[package]]
name = "six"
version = "1.16.0"
description = "Python 2 and 3 compatibility utilities"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
files = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
[[package]]
name = "tiktoken"
version = "0.5.1"
description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -810,6 +929,7 @@ blobfile = ["blobfile (>=2)"]
name = "tokenizers"
version = "0.14.0"
description = ""
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -925,6 +1045,7 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"]
name = "tqdm"
version = "4.66.1"
description = "Fast, Extensible Progress Meter"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -945,6 +1066,7 @@ telegram = ["requests"]
name = "typing-extensions"
version = "4.8.0"
description = "Backported and Experimental Type Hints for Python 3.8+"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -954,25 +1076,26 @@ files = [
[[package]]
name = "urllib3"
version = "2.0.5"
version = "1.26.16"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=3.7"
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
files = [
{file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"},
{file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"},
{file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"},
{file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"},
]
[package.extras]
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"]
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
zstd = ["zstandard (>=0.18.0)"]
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "yarl"
version = "1.9.2"
description = "Yet another URL library"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -1060,6 +1183,7 @@ multidict = ">=4.0"
name = "zipp"
version = "3.17.0"
description = "Backport of pathlib-compatible object wrapper for zip files"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -1074,4 +1198,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = "^3.8"
content-hash = "0fa234d1342838a6cc444dd996dbe404ca2cd6c872dcf560dbe420a2956aaecd"
content-hash = "cb4631107fb44b45122aff1ee0f2657d5e1bf32bec0882f79690473e25ea2b37"

1
pyproject.toml
View file

@ -14,6 +14,7 @@ tiktoken = ">=0.4.0"
importlib-metadata = ">=6.8.0"
tokenizers = "*"
click = "*"
boto3 = ">=1.28.57"
[tool.poetry.scripts]
litellm = 'litellm:run_server'