phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) sagemaker Llama-2 70b

Browse source
This commit is contained in:
ishaan-jaff 2023-12-05 15:32:15 -08:00
parent 68ca2a28d4
commit c4bda13820
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/utils.py
View file

@ -2206,7 +2206,9 @@ def get_optional_params( # use the openai defaults
if max_tokens is not None: if max_tokens is not None:
optional_params["max_output_tokens"] = max_tokens optional_params["max_output_tokens"] = max_tokens
elif custom_llm_provider == "sagemaker": elif custom_llm_provider == "sagemaker":
if "llama-2" in model: if "llama-2" in model.lower() or (
"llama" in model.lower() and "2" in model.lower() # some combination of llama and "2" should exist
): # jumpstart can also send "Llama-2-70b-chat-hf-48xlarge"
# llama-2 models on sagemaker support the following args # llama-2 models on sagemaker support the following args
""" """
max_new_tokens: Model generates text until the output length (excluding the input context length) reaches max_new_tokens. If specified, it must be a positive integer. max_new_tokens: Model generates text until the output length (excluding the input context length) reaches max_new_tokens. If specified, it must be a positive integer.