diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c4294a34..f52c8a68c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,8 +32,7 @@ jobs: pip install -q google-generativeai pip install openai[datalib] pip install -Uq chromadb==0.3.29 - pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl - pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl + pip install "boto3>=1.28.57" - save_cache: paths: - ./venv diff --git a/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl b/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl deleted file mode 100644 index 7d630950a..000000000 Binary files a/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl and /dev/null differ diff --git a/cookbook/bedrock_resources/boto3-1.28.21.tar.gz b/cookbook/bedrock_resources/boto3-1.28.21.tar.gz deleted file mode 100644 index 389c8dc32..000000000 Binary files a/cookbook/bedrock_resources/boto3-1.28.21.tar.gz and /dev/null differ diff --git a/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl b/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl deleted file mode 100644 index 9da7abfa7..000000000 Binary files a/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl and /dev/null differ diff --git a/cookbook/bedrock_resources/botocore-1.31.21.tar.gz b/cookbook/bedrock_resources/botocore-1.31.21.tar.gz deleted file mode 100644 index ab801904b..000000000 Binary files a/cookbook/bedrock_resources/botocore-1.31.21.tar.gz and /dev/null differ diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index e65ac5500..74ade1de8 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -75,13 +75,3 @@ for chunk in response: } } ``` - -## Troubleshooting -If creating a boto3 bedrock client fails with `Unknown service: 'bedrock'` -Try re installing boto3 using the following commands -```shell -pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl -pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl -``` - -See Page 26 on [Amazon Bedrock User Guide](https://d2eo22ngex1n9g.cloudfront.net/Documentation/BedrockUserGuide.pdf) \ No newline at end of file diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 0e19d9054..7286ae873 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -1,11 +1,8 @@ -import os import json from enum import Enum -import requests import time from typing import Callable from litellm.utils import ModelResponse, get_secret -import sys class BedrockError(Exception): def __init__(self, status_code, message): @@ -15,38 +12,24 @@ class BedrockError(Exception): self.message ) # Call the base class constructor with the parameters it needs + class AnthropicConstants(Enum): HUMAN_PROMPT = "\n\nHuman:" AI_PROMPT = "\n\nAssistant:" -def init_bedrock_client(region_name): - import sys - import boto3 - import subprocess - try: - client = boto3.client( - service_name="bedrock", - region_name=region_name, - endpoint_url=f'https://bedrock.{region_name}.amazonaws.com' - ) - except Exception as e: - try: - command1 = "python3 -m pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl" - subprocess.run(command1, shell=True, check=True) - # Command 2: Install boto3 from URL - command2 = "python3 -m pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl" - subprocess.run(command2, shell=True, check=True) - import boto3 - client = boto3.client( - service_name="bedrock", - region_name=region_name, - endpoint_url=f'https://bedrock.{region_name}.amazonaws.com' - ) - except Exception as e: - raise e +def init_bedrock_client(region_name): + import boto3 + + client = boto3.client( + service_name="bedrock-runtime", + region_name=region_name, + endpoint_url=f'https://bedrock-runtime.{region_name}.amazonaws.com' + ) + return client + def convert_messages_to_prompt(messages, provider): # handle anthropic prompts using anthropic constants if provider == "anthropic": @@ -80,30 +63,31 @@ def convert_messages_to_prompt(messages, provider): prompt += f"{message['content']}" return prompt + """ BEDROCK AUTH Keys/Vars os.environ['AWS_ACCESS_KEY_ID'] = "" os.environ['AWS_SECRET_ACCESS_KEY'] = "" """ + # set os.environ['AWS_REGION_NAME'] = def completion( - model: str, - messages: list, - model_response: ModelResponse, - print_verbose: Callable, - encoding, - logging_obj, - optional_params=None, - stream=False, - litellm_params=None, - logger_fn=None, + model: str, + messages: list, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj, + optional_params=None, + stream=False, + litellm_params=None, + logger_fn=None, ): - region_name = ( - get_secret("AWS_REGION_NAME") or - "us-west-2" # default to us-west-2 if user not specified + get_secret("AWS_REGION_NAME") or + "us-west-2" # default to us-west-2 if user not specified ) client = init_bedrock_client(region_name) @@ -119,48 +103,48 @@ def completion( elif provider == "ai21": data = json.dumps({ "prompt": prompt, - }) + }) - else: # amazon titan + else: # amazon titan data = json.dumps({ - "inputText": prompt, + "inputText": prompt, "textGenerationConfig": optional_params, - }) - ## LOGGING + }) + ## LOGGING logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={"complete_input_dict": data}, - ) - + input=prompt, + api_key="", + additional_args={"complete_input_dict": data}, + ) + ## COMPLETION CALL accept = 'application/json' contentType = 'application/json' if stream == True: response = client.invoke_model_with_response_stream( - body=data, - modelId=model, - accept=accept, + body=data, + modelId=model, + accept=accept, contentType=contentType ) response = response.get('body') return response response = client.invoke_model( - body=data, - modelId=model, - accept=accept, + body=data, + modelId=model, + accept=accept, contentType=contentType ) response_body = json.loads(response.get('body').read()) ## LOGGING logging_obj.post_call( - input=prompt, - api_key="", - original_response=response, - additional_args={"complete_input_dict": data}, - ) + input=prompt, + api_key="", + original_response=response, + additional_args={"complete_input_dict": data}, + ) print_verbose(f"raw model_response: {response}") ## RESPONSE OBJECT outputText = "default" @@ -169,7 +153,7 @@ def completion( elif provider == "anthropic": outputText = response_body['completion'] model_response["finish_reason"] = response_body["stop_reason"] - else: # amazon titan + else: # amazon titan outputText = response_body.get('results')[0].get('outputText') if "error" in outputText: raise BedrockError( @@ -185,7 +169,7 @@ def completion( ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here. prompt_tokens = len( encoding.encode(prompt) - ) + ) completion_tokens = len( encoding.encode(model_response["choices"][0]["message"]["content"]) ) @@ -199,6 +183,7 @@ def completion( } return model_response + def embedding(): # logic for parsing in - calling - parsing out model embedding calls pass diff --git a/poetry.lock b/poetry.lock index dc313bd6f..aafdd5115 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "aiohttp" version = "3.8.5" description = "Async http client/server framework (asyncio)" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -112,6 +113,7 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -126,6 +128,7 @@ frozenlist = ">=1.1.0" name = "async-timeout" version = "4.0.3" description = "Timeout context manager for asyncio programs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -137,6 +140,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -151,10 +155,51 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +[[package]] +name = "boto3" +version = "1.28.57" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "boto3-1.28.57-py3-none-any.whl", hash = "sha256:5ddf24cf52c7fb6aaa332eaa08ae8c2afc8f2d1e8860680728533dd573904e32"}, + {file = "boto3-1.28.57.tar.gz", hash = "sha256:e2d2824ba6459b330d097e94039a9c4f96ae3f4bcdc731d620589ad79dcd16d3"}, +] + +[package.dependencies] +botocore = ">=1.31.57,<1.32.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.7.0,<0.8.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.31.57" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "botocore-1.31.57-py3-none-any.whl", hash = "sha256:af006248276ff8e19e3ec7214478f6257035eb40aed865e405486500471ae71b"}, + {file = "botocore-1.31.57.tar.gz", hash = "sha256:301436174635bec739b225b840fc365ca00e5c1a63e5b2a19ee679d204e01b78"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.16.26)"] + [[package]] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -166,6 +211,7 @@ files = [ name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -250,6 +296,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -264,6 +311,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -275,6 +323,7 @@ files = [ name = "filelock" version = "3.12.4" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -291,6 +340,7 @@ typing = ["typing-extensions (>=4.7.1)"] name = "frozenlist" version = "1.4.0" description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -361,6 +411,7 @@ files = [ name = "fsspec" version = "2023.9.2" description = "File-system specification" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -396,6 +447,7 @@ tqdm = ["tqdm"] name = "huggingface-hub" version = "0.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -428,6 +480,7 @@ typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "typ name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -439,6 +492,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -454,10 +508,23 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker perf = ["ipython"] testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "multidict" version = "6.0.4" description = "multidict implementation" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -541,6 +608,7 @@ files = [ name = "openai" version = "0.28.1" description = "Python client library for the OpenAI API" +category = "main" optional = false python-versions = ">=3.7.1" files = [ @@ -555,7 +623,7 @@ tqdm = "*" [package.extras] datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] -dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"] +dev = ["black (>=21.6b0,<22.0)", "pytest (>=6.0.0,<7.0.0)", "pytest-asyncio", "pytest-mock"] embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"] wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"] @@ -563,6 +631,7 @@ wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1 name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -570,10 +639,26 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -588,6 +673,7 @@ cli = ["click (>=5.0)"] name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -647,6 +733,7 @@ files = [ name = "regex" version = "2023.8.8" description = "Alternative regular expression module, to replace re." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -744,6 +831,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -761,10 +849,41 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "s3transfer" +version = "0.7.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.7.0-py3-none-any.whl", hash = "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a"}, + {file = "s3transfer-0.7.0.tar.gz", hash = "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "tiktoken" version = "0.5.1" description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -810,6 +929,7 @@ blobfile = ["blobfile (>=2)"] name = "tokenizers" version = "0.14.0" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -925,6 +1045,7 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] name = "tqdm" version = "4.66.1" description = "Fast, Extensible Progress Meter" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -945,6 +1066,7 @@ telegram = ["requests"] name = "typing-extensions" version = "4.8.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -954,25 +1076,26 @@ files = [ [[package]] name = "urllib3" -version = "2.0.5" +version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, - {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "yarl" version = "1.9.2" description = "Yet another URL library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1060,6 +1183,7 @@ multidict = ">=4.0" name = "zipp" version = "3.17.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1074,4 +1198,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "0fa234d1342838a6cc444dd996dbe404ca2cd6c872dcf560dbe420a2956aaecd" +content-hash = "cb4631107fb44b45122aff1ee0f2657d5e1bf32bec0882f79690473e25ea2b37" diff --git a/pyproject.toml b/pyproject.toml index 7186e84d7..ef7e4ca56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ tiktoken = ">=0.4.0" importlib-metadata = ">=6.8.0" tokenizers = "*" click = "*" +boto3 = ">=1.28.57" [tool.poetry.scripts] litellm = 'litellm:run_server'