phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-05 10:13:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add JSON structured outputs to Ollama

Browse source
This commit is contained in:
Aidan Do 2024-12-22 15:47:39 +11:00
parent bae197c37e
commit da82fb22f6
2 changed files with 15 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

17
llama_stack/providers/remote/inference/ollama/ollama.py
View file

@ -48,10 +48,10 @@ model_aliases = [
"llama3.1:8b-instruct-fp16", "llama3.1:8b-instruct-fp16",
CoreModelId.llama3_1_8b_instruct.value, CoreModelId.llama3_1_8b_instruct.value,
), ),
build_model_alias_with_just_provider_model_id( # build_model_alias_with_just_provider_model_id(
"llama3.1:8b", # "llama3.1:8b",
CoreModelId.llama3_1_8b_instruct.value, # CoreModelId.llama3_1_8b_instruct.value,
), # ),
build_model_alias( build_model_alias(
"llama3.1:70b-instruct-fp16", "llama3.1:70b-instruct-fp16",
CoreModelId.llama3_1_70b_instruct.value, CoreModelId.llama3_1_70b_instruct.value,
@ -214,6 +214,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate):
tool_prompt_format=tool_prompt_format, tool_prompt_format=tool_prompt_format,
stream=stream, stream=stream,
logprobs=logprobs, logprobs=logprobs,
response_format=response_format,
) )
if stream: if stream:
return self._stream_chat_completion(request) return self._stream_chat_completion(request)
@ -257,6 +258,14 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate):
) )
input_dict["raw"] = True input_dict["raw"] = True
if fmt := request.response_format:
if fmt.type == "json_schema":
input_dict["format"] = fmt.json_schema
elif fmt.type == "grammar":
raise NotImplementedError("Grammar response format is not supported")
else:
raise ValueError(f"Unknown response format type: {fmt.type}")
return { return {
"model": request.model, "model": request.model,
**input_dict, **input_dict,

2
llama_stack/providers/tests/inference/test_text_inference.py
View file

@ -191,6 +191,7 @@ class TestInference:
provider = inference_impl.routing_table.get_provider_impl(inference_model) provider = inference_impl.routing_table.get_provider_impl(inference_model)
if provider.__provider_spec__.provider_type not in ( if provider.__provider_spec__.provider_type not in (
"inline::meta-reference", "inline::meta-reference",
"remote::ollama",
"remote::tgi", "remote::tgi",
"remote::together", "remote::together",
"remote::fireworks", "remote::fireworks",
@ -253,6 +254,7 @@ class TestInference:
provider = inference_impl.routing_table.get_provider_impl(inference_model) provider = inference_impl.routing_table.get_provider_impl(inference_model)
if provider.__provider_spec__.provider_type not in ( if provider.__provider_spec__.provider_type not in (
"inline::meta-reference", "inline::meta-reference",
"remote::ollama",
"remote::fireworks", "remote::fireworks",
"remote::tgi", "remote::tgi",
"remote::together", "remote::together",