mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-27 11:43:54 +00:00
fix(health.md): add background health check details to docs
This commit is contained in:
parent
c4e28a9250
commit
87df233a19
6 changed files with 154 additions and 186 deletions
|
@ -248,63 +248,3 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj):
|
|||
return model_response
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
|
||||
# ollama implementation
|
||||
@async_generator
|
||||
async def async_get_ollama_response_stream(
|
||||
api_base="http://localhost:11434",
|
||||
model="llama2",
|
||||
prompt="Why is the sky blue?",
|
||||
optional_params=None,
|
||||
logging_obj=None,
|
||||
):
|
||||
url = f"{api_base}/api/generate"
|
||||
|
||||
## Load Config
|
||||
config=litellm.OllamaConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if k not in optional_params: # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
data = {
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
**optional_params
|
||||
}
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=None,
|
||||
api_key=None,
|
||||
additional_args={"api_base": url, "complete_input_dict": data},
|
||||
)
|
||||
session = requests.Session()
|
||||
|
||||
with session.post(url, json=data, stream=True) as resp:
|
||||
if resp.status_code != 200:
|
||||
raise OllamaError(status_code=resp.status_code, message=resp.text)
|
||||
for line in resp.iter_lines():
|
||||
if line:
|
||||
try:
|
||||
json_chunk = line.decode("utf-8")
|
||||
chunks = json_chunk.split("\n")
|
||||
for chunk in chunks:
|
||||
if chunk.strip() != "":
|
||||
j = json.loads(chunk)
|
||||
if "error" in j:
|
||||
completion_obj = {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"error": j
|
||||
}
|
||||
await yield_({"choices": [{"delta": completion_obj}]})
|
||||
if "response" in j:
|
||||
completion_obj = {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
}
|
||||
completion_obj["content"] = j["response"]
|
||||
await yield_({"choices": [{"delta": completion_obj}]})
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.debug(f"Error decoding JSON: {e}")
|
||||
session.close()
|
Loading…
Add table
Add a link
Reference in a new issue