fix(health.md): add background health check details to docs

This commit is contained in:
Krrish Dholakia 2023-12-16 10:31:46 -08:00
parent c4e28a9250
commit 87df233a19
6 changed files with 154 additions and 186 deletions

View file

@ -248,63 +248,3 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj):
return model_response
except Exception as e:
traceback.print_exc()
# ollama implementation
@async_generator
async def async_get_ollama_response_stream(
api_base="http://localhost:11434",
model="llama2",
prompt="Why is the sky blue?",
optional_params=None,
logging_obj=None,
):
url = f"{api_base}/api/generate"
## Load Config
config=litellm.OllamaConfig.get_config()
for k, v in config.items():
if k not in optional_params: # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
data = {
"model": model,
"prompt": prompt,
**optional_params
}
## LOGGING
logging_obj.pre_call(
input=None,
api_key=None,
additional_args={"api_base": url, "complete_input_dict": data},
)
session = requests.Session()
with session.post(url, json=data, stream=True) as resp:
if resp.status_code != 200:
raise OllamaError(status_code=resp.status_code, message=resp.text)
for line in resp.iter_lines():
if line:
try:
json_chunk = line.decode("utf-8")
chunks = json_chunk.split("\n")
for chunk in chunks:
if chunk.strip() != "":
j = json.loads(chunk)
if "error" in j:
completion_obj = {
"role": "assistant",
"content": "",
"error": j
}
await yield_({"choices": [{"delta": completion_obj}]})
if "response" in j:
completion_obj = {
"role": "assistant",
"content": "",
}
completion_obj["content"] = j["response"]
await yield_({"choices": [{"delta": completion_obj}]})
except Exception as e:
import logging
logging.debug(f"Error decoding JSON: {e}")
session.close()