phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

remove cuda from petals

Browse source
This commit is contained in:
ishaan-jaff 2023-09-20 09:23:23 -07:00
parent 102fd83ea6
commit b81f8d2ddd
2 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

3
litellm/llms/petals.py
View file

@ -39,7 +39,6 @@ def completion(
tokenizer = AutoTokenizer.from_pretrained(model, use_fast=False, add_bos_token=False) tokenizer = AutoTokenizer.from_pretrained(model, use_fast=False, add_bos_token=False)
model_obj = AutoDistributedModelForCausalLM.from_pretrained(model) model_obj = AutoDistributedModelForCausalLM.from_pretrained(model)
model_obj = model_obj.cuda()
prompt = "" prompt = ""
for message in messages: for message in messages:
@ -64,7 +63,7 @@ def completion(
) )
## COMPLETION CALL ## COMPLETION CALL
inputs = tokenizer(prompt, return_tensors="pt")["input_ids"].cuda() inputs = tokenizer(prompt, return_tensors="pt")["input_ids"]
# optional params: max_new_tokens=1,temperature=0.9, top_p=0.6 # optional params: max_new_tokens=1,temperature=0.9, top_p=0.6
outputs = model_obj.generate(inputs, **optional_params) outputs = model_obj.generate(inputs, **optional_params)

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "litellm" name = "litellm"
version = "0.1.714" version = "0.1.715"
description = "Library to easily interface with LLM API providers" description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"] authors = ["BerriAI"]
license = "MIT License" license = "MIT License"