Allow setting environment variables from llama stack run and fix ollama

This commit is contained in:
Ashwin Bharambe 2024-11-17 19:33:48 -08:00
parent a061f3f8c1
commit b1d119466e
19 changed files with 129 additions and 55 deletions

View file

@ -2,7 +2,7 @@ version: '2'
name: ollama
distribution_spec:
description: Use (an external) Ollama server for running LLM inference
docker_image: llamastack/distribution-ollama:test-0.0.52rc3
docker_image: null
providers:
inference:
- remote::ollama

View file

@ -23,9 +23,7 @@ def get_distribution_template() -> DistributionTemplate:
inference_provider = Provider(
provider_id="ollama",
provider_type="remote::ollama",
config=OllamaImplConfig.sample_run_config(
port_str="${env.OLLAMA_PORT}",
),
config=OllamaImplConfig.sample_run_config(),
)
inference_model = ModelInput(
@ -41,7 +39,7 @@ def get_distribution_template() -> DistributionTemplate:
name="ollama",
distro_type="self_hosted",
description="Use (an external) Ollama server for running LLM inference",
docker_image="llamastack/distribution-ollama:test-0.0.52rc3",
docker_image=None,
template_path=Path(__file__).parent / "doc_template.md",
providers=providers,
default_models=[inference_model, safety_model],
@ -74,9 +72,9 @@ def get_distribution_template() -> DistributionTemplate:
"meta-llama/Llama-3.2-3B-Instruct",
"Inference model loaded into the TGI server",
),
"OLLAMA_PORT": (
"14343",
"Port of the Ollama server",
"OLLAMA_URL": (
"http://host.docker.internal:11434",
"URL of the Ollama server",
),
"SAFETY_MODEL": (
"meta-llama/Llama-Guard-3-1B",

View file

@ -87,7 +87,7 @@ class RunConfigSettings(BaseModel):
return StackRunConfig(
image_name=name,
docker_image=docker_image,
built_at=datetime.now(),
built_at=datetime.now().strftime("%Y-%m-%d %H:%M"),
apis=list(apis),
providers=provider_configs,
metadata_store=SqliteKVStoreConfig.sample_run_config(