phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

add top_k + stop to litellm utils

Browse source
This commit is contained in:
ishaan-jaff 2023-09-15 08:34:00 -07:00
parent c0e464a2f6
commit b75b4bcf21
2 changed files with 14 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

13
docs/my-website/docs/providers/togetherai.md
View file

@ -18,7 +18,7 @@ os.environ["TOGETHERAI_API_KEY"] = ""
messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] messages = [{"role": "user", "content": "Write me a poem about the blue sky"}]
completion(model="togethercomputer/Llama-2-7B-32K-Instruct", messages=messages, custom_llm_provider="together_ai") completion(model="togethercomputer/Llama-2-7B-32K-Instruct", messages=messages)
``` ```
### Together AI Models ### Together AI Models
@ -29,10 +29,15 @@ Example TogetherAI Usage - Note: liteLLM supports all models deployed on Togethe
| Model Name | Function Call | Required OS Variables | | Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------| |-----------------------------------|------------------------------------------------------------------------|---------------------------------|
| togethercomputer/llama-2-70b-chat | `completion('togethercomputer/llama-2-70b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` | | togethercomputer/llama-2-70b-chat | `completion('togethercomputer/llama-2-70b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/llama-2-70b | `completion('togethercomputer/llama-2-70b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/LLaMA-2-13b-chat | `completion('togethercomputer/LLaMA-2-13b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` | | togethercomputer/LLaMA-2-13b-chat | `completion('togethercomputer/LLaMA-2-13b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/code-and-talk-v1 | `completion('togethercomputer/code-and-talk-v1', messages)` | `os.environ['TOGETHERAI_API_KEY']` | | togethercomputer/LLaMA-2-7B-32K | `completion('togethercomputer/LLaMA-2-7B-32K', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/creative-v1 | `completion('togethercomputer/creative-v1', messages)` | `os.environ['TOGETHERAI_API_KEY']` | | togethercomputer/Llama-2-7B-32K-Instruct | `completion('togethercomputer/Llama-2-7B-32K-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/yourmodel | `completion('togethercomputer/yourmodel', messages)` | `os.environ['TOGETHERAI_API_KEY']` | | togethercomputer/llama-2-7b | `completion('togethercomputer/llama-2-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/CodeLlama-34b | `completion('togethercomputer/CodeLlama-34b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/CodeLlama-34b-Instruct | `completion('togethercomputer/CodeLlama-34b-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/CodeLlama-34b-Python | `completion('togethercomputer/CodeLlama-34b-Python', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
### Prompt Templates ### Prompt Templates

6
litellm/utils.py
View file

@ -875,10 +875,14 @@ def get_optional_params( # use the openai defaults
optional_params["temperature"] = temperature optional_params["temperature"] = temperature
if top_p != 1: if top_p != 1:
optional_params["top_p"] = top_p optional_params["top_p"] = top_p
if top_k != 40:
optional_params["top_k"] = top_k
if max_tokens != float("inf"): if max_tokens != float("inf"):
optional_params["max_tokens"] = max_tokens optional_params["max_tokens"] = max_tokens
if frequency_penalty != 0: if frequency_penalty != 0:
optional_params["frequency_penalty"] = frequency_penalty optional_params["frequency_penalty"] = frequency_penalty # should be repetition penalty
if stop != None:
optional_params["stop"] = stop #TG AI expects a list, example ["\n\n\n\n","<|endoftext|>"]
elif ( elif (
model == "chat-bison" model == "chat-bison"
): # chat-bison has diff args from chat-bison@001 ty Google ): # chat-bison has diff args from chat-bison@001 ty Google