From 0ddda7c0359fd6abb7c74224f2d00256ee28a047 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Sep 2023 11:01:37 -0700 Subject: [PATCH] send optional_params for llama2-70b chat replicate --- litellm/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index dd40cfa63..1a14fbe93 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -702,13 +702,20 @@ def get_optional_params( # use the openai defaults optional_params["logit_bias"] = logit_bias return optional_params elif custom_llm_provider == "replicate": - # any replicate models - # TODO: handle translating remaining replicate params if stream: optional_params["stream"] = stream return optional_params if max_tokens != float("inf"): optional_params["max_new_tokens"] = max_tokens + if temperature != 1: + optional_params["temperature"] = temperature + if top_p != 1: + optional_params["top_p"] = top_p + if top_k != 40: + optional_params["top_k"] = top_k + if stop != None: + optional_params["stop_sequences"] = stop + elif custom_llm_provider == "together_ai" or ("togethercomputer" in model): if stream: optional_params["stream_tokens"] = stream