diff --git a/llama_stack/distribution/datatypes.py b/llama_stack/distribution/datatypes.py index ead1331f3..45c42893f 100644 --- a/llama_stack/distribution/datatypes.py +++ b/llama_stack/distribution/datatypes.py @@ -146,11 +146,10 @@ in the runtime configuration to help route to the correct provider.""", class Provider(BaseModel): - # provider_id of None means that the provider is not enabled - this happens - # when the provider is enabled via a conditional environment variable - provider_id: str | None + provider_id: str provider_type: str config: dict[str, Any] + enabled: bool = Field(default=True, description="Whether the provider is enabled") class LoggingConfig(BaseModel): diff --git a/llama_stack/distribution/resolver.py b/llama_stack/distribution/resolver.py index 46cd1161e..7773e573f 100644 --- a/llama_stack/distribution/resolver.py +++ b/llama_stack/distribution/resolver.py @@ -199,7 +199,7 @@ def validate_and_prepare_providers( specs = {} for provider in providers: - if not provider.provider_id or provider.provider_id == "__disabled__": + if not provider.provider_id or not provider.enabled: logger.warning(f"Provider `{provider.provider_type}` for API `{api}` is disabled") continue diff --git a/llama_stack/distribution/stack.py b/llama_stack/distribution/stack.py index 1a9237d6c..5d2393785 100644 --- a/llama_stack/distribution/stack.py +++ b/llama_stack/distribution/stack.py @@ -99,19 +99,10 @@ async def register_resources(run_config: StackRunConfig, impls: dict[Api, Any]): method = getattr(impls[api], register_method) for obj in objects: # Do not register models on disabled providers - if hasattr(obj, "provider_id") and obj.provider_id is not None and obj.provider_id == "__disabled__": - logger.debug(f"Skipping {rsrc.capitalize()} registration for disabled provider.") - continue - # In complex templates, like our starter template, we may have dynamic model ids - # given by environment variables. This allows those environment variables to have - # a default value of __disabled__ to skip registration of the model if not set. - if ( - hasattr(obj, "provider_model_id") - and obj.provider_model_id is not None - and "__disabled__" in obj.provider_model_id - ): + if hasattr(obj, "provider_model_id") and obj.provider_model_id is not None and not obj.enabled: logger.debug(f"Skipping {rsrc.capitalize()} registration for disabled model.") continue + # we want to maintain the type information in arguments to method. # instead of method(**obj.model_dump()), which may convert a typed attr to a dict, # we use model_dump() to find all the attrs and then getattr to get the still typed value. @@ -155,17 +146,20 @@ def replace_env_vars(config: Any, path: str = "") -> Any: for i, v in enumerate(config): try: # Special handling for providers: first resolve the provider_id to check if provider - # is disabled so that we can skip config env variable expansion and avoid validation errors + # is disabled so that we can skip config env variable expansion and avoid validation + # errors if isinstance(v, dict) and "provider_id" in v: try: - resolved_provider_id = replace_env_vars(v["provider_id"], f"{path}[{i}].provider_id") - if resolved_provider_id == "__disabled__": + # We have to set a default to True because we use Pydantic + # exclude_defaults=True from the serializer so the loaded config only has + # 'enabled' field when it is set to False explicitly. + if not v.get("enabled", True): logger.debug( - f"Skipping config env variable expansion for disabled provider: {v.get('provider_id', '')}" + f"Skipping config env variable expansion for disabled provider: {v.get('provider_type', '') + '/' if v.get('provider_type', '') else ''}{v.get('provider_id', '')}" ) # Create a copy with resolved provider_id but original config disabled_provider = v.copy() - disabled_provider["provider_id"] = resolved_provider_id + disabled_provider["provider_id"] = v["provider_id"] result.append(disabled_provider) continue except EnvVarError: diff --git a/llama_stack/providers/inline/post_training/huggingface/config.py b/llama_stack/providers/inline/post_training/huggingface/config.py index 06c6d8073..c90221339 100644 --- a/llama_stack/providers/inline/post_training/huggingface/config.py +++ b/llama_stack/providers/inline/post_training/huggingface/config.py @@ -68,5 +68,5 @@ class HuggingFacePostTrainingConfig(BaseModel): dataloader_pin_memory: bool = True @classmethod - def sample_run_config(cls, __distro_dir__: str, **kwargs: Any) -> dict[str, Any]: + def sample_run_config(cls, **kwargs: Any) -> dict[str, Any]: return {"checkpoint_format": "huggingface", "distributed_backend": None, "device": "cpu"} diff --git a/llama_stack/providers/utils/inference/model_registry.py b/llama_stack/providers/utils/inference/model_registry.py index 46c0ca7b5..fc8066eed 100644 --- a/llama_stack/providers/utils/inference/model_registry.py +++ b/llama_stack/providers/utils/inference/model_registry.py @@ -25,6 +25,7 @@ class ProviderModelEntry(BaseModel): llama_model: str | None = None model_type: ModelType = ModelType.llm metadata: dict[str, Any] = Field(default_factory=dict) + enabled: bool = Field(default=True, description="Whether the model is enabled") def get_huggingface_repo(model_descriptor: str) -> str | None: diff --git a/llama_stack/templates/starter/run.yaml b/llama_stack/templates/starter/run.yaml index 02288da44..5ecbc689c 100644 --- a/llama_stack/templates/starter/run.yaml +++ b/llama_stack/templates/starter/run.yaml @@ -14,155 +14,183 @@ apis: - vector_io providers: inference: - - provider_id: ${env.ENABLE_CEREBRAS:=__disabled__} + - provider_id: cerebras provider_type: remote::cerebras config: base_url: https://api.cerebras.ai api_key: ${env.CEREBRAS_API_KEY} - - provider_id: ${env.ENABLE_OLLAMA:=__disabled__} + enabled: false + - provider_id: ollama provider_type: remote::ollama config: url: ${env.OLLAMA_URL:=http://localhost:11434} - - provider_id: ${env.ENABLE_VLLM:=__disabled__} + enabled: false + - provider_id: vllm provider_type: remote::vllm config: url: ${env.VLLM_URL} max_tokens: ${env.VLLM_MAX_TOKENS:=4096} api_token: ${env.VLLM_API_TOKEN:=fake} tls_verify: ${env.VLLM_TLS_VERIFY:=true} - - provider_id: ${env.ENABLE_TGI:=__disabled__} + enabled: false + - provider_id: tgi provider_type: remote::tgi config: url: ${env.TGI_URL} - - provider_id: ${env.ENABLE_HF_SERVERLESS:=__disabled__} + enabled: false + - provider_id: hf::serverless provider_type: remote::hf::serverless config: huggingface_repo: ${env.INFERENCE_MODEL} api_token: ${env.HF_API_TOKEN} - - provider_id: ${env.ENABLE_HF_ENDPOINT:=__disabled__} + enabled: false + - provider_id: hf::endpoint provider_type: remote::hf::endpoint config: endpoint_name: ${env.INFERENCE_ENDPOINT_NAME} api_token: ${env.HF_API_TOKEN} - - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + enabled: false + - provider_id: fireworks provider_type: remote::fireworks config: url: https://api.fireworks.ai/inference/v1 api_key: ${env.FIREWORKS_API_KEY} - - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + enabled: false + - provider_id: together provider_type: remote::together config: url: https://api.together.xyz/v1 api_key: ${env.TOGETHER_API_KEY} - - provider_id: ${env.ENABLE_BEDROCK:=__disabled__} + enabled: false + - provider_id: bedrock provider_type: remote::bedrock config: {} - - provider_id: ${env.ENABLE_DATABRICKS:=__disabled__} + enabled: false + - provider_id: databricks provider_type: remote::databricks config: url: ${env.DATABRICKS_URL} api_token: ${env.DATABRICKS_API_TOKEN} - - provider_id: ${env.ENABLE_NVIDIA:=__disabled__} + enabled: false + - provider_id: nvidia provider_type: remote::nvidia config: url: ${env.NVIDIA_BASE_URL:=https://integrate.api.nvidia.com} api_key: ${env.NVIDIA_API_KEY:=} append_api_version: ${env.NVIDIA_APPEND_API_VERSION:=True} - - provider_id: ${env.ENABLE_RUNPOD:=__disabled__} + enabled: false + - provider_id: runpod provider_type: remote::runpod config: url: ${env.RUNPOD_URL:=} api_token: ${env.RUNPOD_API_TOKEN} - - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + enabled: false + - provider_id: openai provider_type: remote::openai config: api_key: ${env.OPENAI_API_KEY} - - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + enabled: false + - provider_id: anthropic provider_type: remote::anthropic config: api_key: ${env.ANTHROPIC_API_KEY} - - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + enabled: false + - provider_id: gemini provider_type: remote::gemini config: api_key: ${env.GEMINI_API_KEY} - - provider_id: ${env.ENABLE_GROQ:=__disabled__} + enabled: false + - provider_id: groq provider_type: remote::groq config: url: https://api.groq.com api_key: ${env.GROQ_API_KEY} - - provider_id: ${env.ENABLE_FIREWORKS_OPENAI_COMPAT:=__disabled__} + enabled: false + - provider_id: fireworks-openai-compat provider_type: remote::fireworks-openai-compat config: openai_compat_api_base: https://api.fireworks.ai/inference/v1 api_key: ${env.FIREWORKS_API_KEY} - - provider_id: ${env.ENABLE_LLAMA_OPENAI_COMPAT:=__disabled__} + enabled: false + - provider_id: llama-openai-compat provider_type: remote::llama-openai-compat config: openai_compat_api_base: https://api.llama.com/compat/v1/ api_key: ${env.LLAMA_API_KEY} - - provider_id: ${env.ENABLE_TOGETHER_OPENAI_COMPAT:=__disabled__} + enabled: false + - provider_id: together-openai-compat provider_type: remote::together-openai-compat config: openai_compat_api_base: https://api.together.xyz/v1 api_key: ${env.TOGETHER_API_KEY} - - provider_id: ${env.ENABLE_GROQ_OPENAI_COMPAT:=__disabled__} + enabled: false + - provider_id: groq-openai-compat provider_type: remote::groq-openai-compat config: openai_compat_api_base: https://api.groq.com/openai/v1 api_key: ${env.GROQ_API_KEY} - - provider_id: ${env.ENABLE_SAMBANOVA_OPENAI_COMPAT:=__disabled__} + enabled: false + - provider_id: sambanova-openai-compat provider_type: remote::sambanova-openai-compat config: openai_compat_api_base: https://api.sambanova.ai/v1 api_key: ${env.SAMBANOVA_API_KEY} - - provider_id: ${env.ENABLE_CEREBRAS_OPENAI_COMPAT:=__disabled__} + enabled: false + - provider_id: cerebras-openai-compat provider_type: remote::cerebras-openai-compat config: openai_compat_api_base: https://api.cerebras.ai/v1 api_key: ${env.CEREBRAS_API_KEY} - - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + enabled: false + - provider_id: sambanova provider_type: remote::sambanova config: url: https://api.sambanova.ai/v1 api_key: ${env.SAMBANOVA_API_KEY} - - provider_id: ${env.ENABLE_PASSTHROUGH:=__disabled__} + enabled: false + - provider_id: passthrough provider_type: remote::passthrough config: url: ${env.PASSTHROUGH_URL} api_key: ${env.PASSTHROUGH_API_KEY} - - provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:=sentence-transformers} + enabled: false + - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} vector_io: - - provider_id: ${env.ENABLE_FAISS:=faiss} + - provider_id: faiss provider_type: inline::faiss config: kvstore: type: sqlite db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/faiss_store.db - - provider_id: ${env.ENABLE_SQLITE_VEC:=__disabled__} + - provider_id: sqlite-vec provider_type: inline::sqlite-vec config: db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/sqlite_vec.db - - provider_id: ${env.ENABLE_MILVUS:=__disabled__} + enabled: false + - provider_id: milvus provider_type: inline::milvus config: db_path: ${env.MILVUS_DB_PATH:=~/.llama/distributions/starter}/milvus.db kvstore: type: sqlite db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/milvus_registry.db - - provider_id: ${env.ENABLE_CHROMADB:=__disabled__} + enabled: false + - provider_id: chromadb provider_type: remote::chromadb config: - url: ${env.CHROMADB_URL:=} - - provider_id: ${env.ENABLE_PGVECTOR:=__disabled__} + url: ${env.CHROMADB_URL} + enabled: false + - provider_id: pgvector provider_type: remote::pgvector config: host: ${env.PGVECTOR_HOST:=localhost} port: ${env.PGVECTOR_PORT:=5432} - db: ${env.PGVECTOR_DB:=} - user: ${env.PGVECTOR_USER:=} - password: ${env.PGVECTOR_PASSWORD:=} + db: ${env.PGVECTOR_DB} + user: ${env.PGVECTOR_USER} + password: ${env.PGVECTOR_PASSWORD} + enabled: false files: - provider_id: meta-reference-files provider_type: inline::localfs @@ -257,642 +285,642 @@ inference_store: db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/inference_store.db models: - metadata: {} - model_id: ${env.ENABLE_OLLAMA:=__disabled__}/${env.OLLAMA_INFERENCE_MODEL:=__disabled__} - provider_id: ${env.ENABLE_OLLAMA:=__disabled__} - provider_model_id: ${env.OLLAMA_INFERENCE_MODEL:=__disabled__} + model_id: ollama/${env.OLLAMA_INFERENCE_MODEL:=} + provider_id: ollama + provider_model_id: ${env.OLLAMA_INFERENCE_MODEL:=} model_type: llm - metadata: embedding_dimension: ${env.OLLAMA_EMBEDDING_DIMENSION:=384} - model_id: ${env.ENABLE_OLLAMA:=__disabled__}/${env.OLLAMA_EMBEDDING_MODEL:=__disabled__} - provider_id: ${env.ENABLE_OLLAMA:=__disabled__} - provider_model_id: ${env.OLLAMA_EMBEDDING_MODEL:=__disabled__} + model_id: ollama/${env.OLLAMA_EMBEDDING_MODEL:=} + provider_id: ollama + provider_model_id: ${env.OLLAMA_EMBEDDING_MODEL:=} model_type: embedding - metadata: {} - model_id: ${env.ENABLE_VLLM:=__disabled__}/${env.VLLM_INFERENCE_MODEL:=__disabled__} - provider_id: ${env.ENABLE_VLLM:=__disabled__} - provider_model_id: ${env.VLLM_INFERENCE_MODEL:=__disabled__} + model_id: vllm/${env.VLLM_INFERENCE_MODEL:=} + provider_id: vllm + provider_model_id: ${env.VLLM_INFERENCE_MODEL:=} model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p1-8b-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p1-8b-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p1-8b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.1-8B-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.1-8B-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p1-8b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p1-70b-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p1-70b-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p1-70b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.1-70B-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.1-70B-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p1-70b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p1-405b-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p1-405b-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p1-405b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.1-405B-Instruct-FP8 - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.1-405B-Instruct-FP8 + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p1-405b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p2-3b-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p2-3b-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p2-3b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.2-3B-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.2-3B-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p2-3b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p2-11b-vision-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p2-11b-vision-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p2-11b-vision-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.2-11B-Vision-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.2-11B-Vision-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p2-11b-vision-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p2-90b-vision-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p2-90b-vision-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p2-90b-vision-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.2-90B-Vision-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.2-90B-Vision-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p2-90b-vision-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-v3p3-70b-instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-v3p3-70b-instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p3-70b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-3.3-70B-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-3.3-70B-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-v3p3-70b-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-guard-3-8b - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-guard-3-8b + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-guard-3-8b model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-Guard-3-8B - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-Guard-3-8B + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-guard-3-8b model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama-guard-3-11b-vision - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama-guard-3-11b-vision + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-guard-3-11b-vision model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-Guard-3-11B-Vision - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-Guard-3-11B-Vision + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama-guard-3-11b-vision model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama4-scout-instruct-basic - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama4-scout-instruct-basic + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama4-scout-instruct-basic model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama4-scout-instruct-basic model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/accounts/fireworks/models/llama4-maverick-instruct-basic - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: accounts/fireworks/models/llama4-maverick-instruct-basic + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama4-maverick-instruct-basic model_type: llm - metadata: {} - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/meta-llama/Llama-4-Maverick-17B-128E-Instruct + provider_id: fireworks provider_model_id: accounts/fireworks/models/llama4-maverick-instruct-basic model_type: llm - metadata: embedding_dimension: 768 context_length: 8192 - model_id: ${env.ENABLE_FIREWORKS:=__disabled__}/nomic-ai/nomic-embed-text-v1.5 - provider_id: ${env.ENABLE_FIREWORKS:=__disabled__} + model_id: fireworks/nomic-ai/nomic-embed-text-v1.5 + provider_id: fireworks provider_model_id: nomic-ai/nomic-embed-text-v1.5 model_type: embedding - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.1-8B-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.1-8B-Instruct + provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.1-70B-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.1-70B-Instruct + provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.1-405B-Instruct-FP8 - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.1-405B-Instruct-FP8 + provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.2-3B-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.2-3B-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.2-3B-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.2-3B-Instruct + provider_id: together provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.2-11B-Vision-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.2-11B-Vision-Instruct + provider_id: together provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.2-90B-Vision-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.2-90B-Vision-Instruct + provider_id: together provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.3-70B-Instruct-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.3-70B-Instruct-Turbo + provider_id: together provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-3.3-70B-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-3.3-70B-Instruct + provider_id: together provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Meta-Llama-Guard-3-8B - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Meta-Llama-Guard-3-8B + provider_id: together provider_model_id: meta-llama/Meta-Llama-Guard-3-8B model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-Guard-3-8B - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-Guard-3-8B + provider_id: together provider_model_id: meta-llama/Meta-Llama-Guard-3-8B model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-Guard-3-11B-Vision-Turbo - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-Guard-3-11B-Vision-Turbo + provider_id: together provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-Guard-3-11B-Vision - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-Guard-3-11B-Vision + provider_id: together provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo model_type: llm - metadata: embedding_dimension: 768 context_length: 8192 - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/togethercomputer/m2-bert-80M-8k-retrieval - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: togethercomputer/m2-bert-80M-8k-retrieval + provider_id: together provider_model_id: togethercomputer/m2-bert-80M-8k-retrieval model_type: embedding - metadata: embedding_dimension: 768 context_length: 32768 - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/togethercomputer/m2-bert-80M-32k-retrieval - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: togethercomputer/m2-bert-80M-32k-retrieval + provider_id: together provider_model_id: togethercomputer/m2-bert-80M-32k-retrieval model_type: embedding - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/together/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-4-Maverick-17B-128E-Instruct + provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm - metadata: {} - model_id: ${env.ENABLE_TOGETHER:=__disabled__}/together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 - provider_id: ${env.ENABLE_TOGETHER:=__disabled__} + model_id: together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/openai/gpt-4o - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4o + provider_id: openai provider_model_id: openai/gpt-4o model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/openai/gpt-4o-mini - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4o-mini + provider_id: openai provider_model_id: openai/gpt-4o-mini model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/openai/chatgpt-4o-latest - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/chatgpt-4o-latest + provider_id: openai provider_model_id: openai/chatgpt-4o-latest model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-3.5-turbo-0125 - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-3.5-turbo-0125 + provider_id: openai provider_model_id: gpt-3.5-turbo-0125 model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-3.5-turbo - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-3.5-turbo + provider_id: openai provider_model_id: gpt-3.5-turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-3.5-turbo-instruct - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-3.5-turbo-instruct + provider_id: openai provider_model_id: gpt-3.5-turbo-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-4 - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4 + provider_id: openai provider_model_id: gpt-4 model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-4-turbo - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4-turbo + provider_id: openai provider_model_id: gpt-4-turbo model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-4o - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4o + provider_id: openai provider_model_id: gpt-4o model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-4o-2024-08-06 - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4o-2024-08-06 + provider_id: openai provider_model_id: gpt-4o-2024-08-06 model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-4o-mini - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4o-mini + provider_id: openai provider_model_id: gpt-4o-mini model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/gpt-4o-audio-preview - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/gpt-4o-audio-preview + provider_id: openai provider_model_id: gpt-4o-audio-preview model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/chatgpt-4o-latest - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/chatgpt-4o-latest + provider_id: openai provider_model_id: chatgpt-4o-latest model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/o1 - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/o1 + provider_id: openai provider_model_id: o1 model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/o1-mini - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/o1-mini + provider_id: openai provider_model_id: o1-mini model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/o3-mini - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/o3-mini + provider_id: openai provider_model_id: o3-mini model_type: llm - metadata: {} - model_id: ${env.ENABLE_OPENAI:=__disabled__}/o4-mini - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/o4-mini + provider_id: openai provider_model_id: o4-mini model_type: llm - metadata: embedding_dimension: 1536 context_length: 8192 - model_id: ${env.ENABLE_OPENAI:=__disabled__}/openai/text-embedding-3-small - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/text-embedding-3-small + provider_id: openai provider_model_id: openai/text-embedding-3-small model_type: embedding - metadata: embedding_dimension: 3072 context_length: 8192 - model_id: ${env.ENABLE_OPENAI:=__disabled__}/openai/text-embedding-3-large - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/text-embedding-3-large + provider_id: openai provider_model_id: openai/text-embedding-3-large model_type: embedding - metadata: embedding_dimension: 1536 context_length: 8192 - model_id: ${env.ENABLE_OPENAI:=__disabled__}/text-embedding-3-small - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/text-embedding-3-small + provider_id: openai provider_model_id: text-embedding-3-small model_type: embedding - metadata: embedding_dimension: 3072 context_length: 8192 - model_id: ${env.ENABLE_OPENAI:=__disabled__}/text-embedding-3-large - provider_id: ${env.ENABLE_OPENAI:=__disabled__} + model_id: openai/text-embedding-3-large + provider_id: openai provider_model_id: text-embedding-3-large model_type: embedding - metadata: {} - model_id: ${env.ENABLE_ANTHROPIC:=__disabled__}/anthropic/claude-3-5-sonnet-latest - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + model_id: anthropic/claude-3-5-sonnet-latest + provider_id: anthropic provider_model_id: anthropic/claude-3-5-sonnet-latest model_type: llm - metadata: {} - model_id: ${env.ENABLE_ANTHROPIC:=__disabled__}/anthropic/claude-3-7-sonnet-latest - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + model_id: anthropic/claude-3-7-sonnet-latest + provider_id: anthropic provider_model_id: anthropic/claude-3-7-sonnet-latest model_type: llm - metadata: {} - model_id: ${env.ENABLE_ANTHROPIC:=__disabled__}/anthropic/claude-3-5-haiku-latest - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + model_id: anthropic/claude-3-5-haiku-latest + provider_id: anthropic provider_model_id: anthropic/claude-3-5-haiku-latest model_type: llm - metadata: embedding_dimension: 1024 context_length: 32000 - model_id: ${env.ENABLE_ANTHROPIC:=__disabled__}/anthropic/voyage-3 - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + model_id: anthropic/voyage-3 + provider_id: anthropic provider_model_id: anthropic/voyage-3 model_type: embedding - metadata: embedding_dimension: 512 context_length: 32000 - model_id: ${env.ENABLE_ANTHROPIC:=__disabled__}/anthropic/voyage-3-lite - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + model_id: anthropic/voyage-3-lite + provider_id: anthropic provider_model_id: anthropic/voyage-3-lite model_type: embedding - metadata: embedding_dimension: 1024 context_length: 32000 - model_id: ${env.ENABLE_ANTHROPIC:=__disabled__}/anthropic/voyage-code-3 - provider_id: ${env.ENABLE_ANTHROPIC:=__disabled__} + model_id: anthropic/voyage-code-3 + provider_id: anthropic provider_model_id: anthropic/voyage-code-3 model_type: embedding - metadata: {} - model_id: ${env.ENABLE_GEMINI:=__disabled__}/gemini/gemini-1.5-flash - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + model_id: gemini/gemini-1.5-flash + provider_id: gemini provider_model_id: gemini/gemini-1.5-flash model_type: llm - metadata: {} - model_id: ${env.ENABLE_GEMINI:=__disabled__}/gemini/gemini-1.5-pro - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + model_id: gemini/gemini-1.5-pro + provider_id: gemini provider_model_id: gemini/gemini-1.5-pro model_type: llm - metadata: {} - model_id: ${env.ENABLE_GEMINI:=__disabled__}/gemini/gemini-2.0-flash - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + model_id: gemini/gemini-2.0-flash + provider_id: gemini provider_model_id: gemini/gemini-2.0-flash model_type: llm - metadata: {} - model_id: ${env.ENABLE_GEMINI:=__disabled__}/gemini/gemini-2.5-flash - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + model_id: gemini/gemini-2.5-flash + provider_id: gemini provider_model_id: gemini/gemini-2.5-flash model_type: llm - metadata: {} - model_id: ${env.ENABLE_GEMINI:=__disabled__}/gemini/gemini-2.5-pro - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + model_id: gemini/gemini-2.5-pro + provider_id: gemini provider_model_id: gemini/gemini-2.5-pro model_type: llm - metadata: embedding_dimension: 768 context_length: 2048 - model_id: ${env.ENABLE_GEMINI:=__disabled__}/gemini/text-embedding-004 - provider_id: ${env.ENABLE_GEMINI:=__disabled__} + model_id: gemini/text-embedding-004 + provider_id: gemini provider_model_id: gemini/text-embedding-004 model_type: embedding - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama3-8b-8192 - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama3-8b-8192 + provider_id: groq provider_model_id: groq/llama3-8b-8192 model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-3.1-8B-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-3.1-8B-Instruct + provider_id: groq provider_model_id: groq/llama3-8b-8192 model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama-3.1-8b-instant - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama-3.1-8b-instant + provider_id: groq provider_model_id: groq/llama-3.1-8b-instant model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama3-70b-8192 - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama3-70b-8192 + provider_id: groq provider_model_id: groq/llama3-70b-8192 model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-3-70B-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-3-70B-Instruct + provider_id: groq provider_model_id: groq/llama3-70b-8192 model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama-3.3-70b-versatile - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama-3.3-70b-versatile + provider_id: groq provider_model_id: groq/llama-3.3-70b-versatile model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-3.3-70B-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-3.3-70B-Instruct + provider_id: groq provider_model_id: groq/llama-3.3-70b-versatile model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama-3.2-3b-preview - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama-3.2-3b-preview + provider_id: groq provider_model_id: groq/llama-3.2-3b-preview model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-3.2-3B-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-3.2-3B-Instruct + provider_id: groq provider_model_id: groq/llama-3.2-3b-preview model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama-4-scout-17b-16e-instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama-4-scout-17b-16e-instruct + provider_id: groq provider_model_id: groq/llama-4-scout-17b-16e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: groq provider_model_id: groq/llama-4-scout-17b-16e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/meta-llama/llama-4-scout-17b-16e-instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/llama-4-scout-17b-16e-instruct + provider_id: groq provider_model_id: groq/meta-llama/llama-4-scout-17b-16e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: groq provider_model_id: groq/meta-llama/llama-4-scout-17b-16e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/llama-4-maverick-17b-128e-instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/llama-4-maverick-17b-128e-instruct + provider_id: groq provider_model_id: groq/llama-4-maverick-17b-128e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-4-Maverick-17B-128E-Instruct + provider_id: groq provider_model_id: groq/llama-4-maverick-17b-128e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/groq/meta-llama/llama-4-maverick-17b-128e-instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/llama-4-maverick-17b-128e-instruct + provider_id: groq provider_model_id: groq/meta-llama/llama-4-maverick-17b-128e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_GROQ:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct - provider_id: ${env.ENABLE_GROQ:=__disabled__} + model_id: groq/meta-llama/Llama-4-Maverick-17B-128E-Instruct + provider_id: groq provider_model_id: groq/meta-llama/llama-4-maverick-17b-128e-instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-3.1-8B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Meta-Llama-3.1-8B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.1-8B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.1-8B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.1-8B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.1-8B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-3.1-405B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Meta-Llama-3.1-405B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.1-405B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.1-405B-Instruct-FP8 - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.1-405B-Instruct-FP8 + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.1-405B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-3.2-1B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Meta-Llama-3.2-1B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.2-1B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.2-1B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.2-1B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.2-1B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-3.2-3B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Meta-Llama-3.2-3B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.2-3B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.2-3B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.2-3B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.2-3B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-3.3-70B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Meta-Llama-3.3-70B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.3-70B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.3-70B-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.3-70B-Instruct + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-3.3-70B-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-3.2-11B-Vision-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Llama-3.2-11B-Vision-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-3.2-11B-Vision-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.2-11B-Vision-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.2-11B-Vision-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-3.2-11B-Vision-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-3.2-90B-Vision-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Llama-3.2-90B-Vision-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-3.2-90B-Vision-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.2-90B-Vision-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-3.2-90B-Vision-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-3.2-90B-Vision-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Llama-4-Scout-17B-16E-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-4-Scout-17B-16E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-4-Scout-17B-16E-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-4-Scout-17B-16E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-4-Maverick-17B-128E-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Llama-4-Maverick-17B-128E-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-4-Maverick-17B-128E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-4-Maverick-17B-128E-Instruct + provider_id: sambanova provider_model_id: sambanova/Llama-4-Maverick-17B-128E-Instruct model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-Guard-3-8B - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/Meta-Llama-Guard-3-8B + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-Guard-3-8B model_type: llm - metadata: {} - model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-Guard-3-8B - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} + model_id: sambanova/meta-llama/Llama-Guard-3-8B + provider_id: sambanova provider_model_id: sambanova/Meta-Llama-Guard-3-8B model_type: llm - metadata: embedding_dimension: 384 model_id: all-MiniLM-L6-v2 - provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:=sentence-transformers} + provider_id: sentence-transformers model_type: embedding shields: [] vector_dbs: [] diff --git a/llama_stack/templates/starter/starter.py b/llama_stack/templates/starter/starter.py index 773693285..2347d8bad 100644 --- a/llama_stack/templates/starter/starter.py +++ b/llama_stack/templates/starter/starter.py @@ -78,22 +78,25 @@ def _get_model_entries_for_provider(provider_type: str) -> list[ProviderModelEnt if provider_type == "ollama": return [ ProviderModelEntry( - provider_model_id="${env.OLLAMA_INFERENCE_MODEL:=__disabled__}", + provider_model_id="${env.OLLAMA_INFERENCE_MODEL:=}", model_type=ModelType.llm, + enabled=False, ), ProviderModelEntry( - provider_model_id="${env.OLLAMA_EMBEDDING_MODEL:=__disabled__}", + provider_model_id="${env.OLLAMA_EMBEDDING_MODEL:=}", model_type=ModelType.embedding, metadata={ "embedding_dimension": "${env.OLLAMA_EMBEDDING_DIMENSION:=384}", }, + enabled=False, ), ] elif provider_type == "vllm": return [ ProviderModelEntry( - provider_model_id="${env.VLLM_INFERENCE_MODEL:=__disabled__}", + provider_model_id="${env.VLLM_INFERENCE_MODEL:=}", model_type=ModelType.llm, + enabled=False, ), ] @@ -129,29 +132,29 @@ def get_remote_inference_providers() -> tuple[list[Provider], dict[str, list[Pro provider_type = provider_spec.adapter.adapter_type # Build the environment variable name for enabling this provider - env_var = f"ENABLE_{provider_type.upper().replace('-', '_').replace('::', '_')}" + # only get the provider type after the :: model_entries = _get_model_entries_for_provider(provider_type) config = _get_config_for_provider(provider_spec) providers.append( ( - f"${{env.{env_var}:=__disabled__}}", provider_type, model_entries, config, ) ) - available_models[f"${{env.{env_var}:=__disabled__}}"] = model_entries + available_models[provider_type] = model_entries inference_providers = [] - for provider_id, provider_type, model_entries, config in providers: + for provider_type, model_entries, config in providers: inference_providers.append( Provider( - provider_id=provider_id, + provider_id=provider_type, provider_type=f"remote::{provider_type}", config=config, + enabled=False, ) ) - available_models[provider_id] = model_entries + available_models[provider_type] = model_entries return inference_providers, available_models @@ -162,33 +165,33 @@ def get_distribution_template() -> DistributionTemplate: vector_io_providers = [ Provider( - provider_id="${env.ENABLE_FAISS:=faiss}", + provider_id="faiss", provider_type="inline::faiss", config=FaissVectorIOConfig.sample_run_config(f"~/.llama/distributions/{name}"), ), Provider( - provider_id="${env.ENABLE_SQLITE_VEC:=__disabled__}", + provider_id="sqlite-vec", provider_type="inline::sqlite-vec", config=SQLiteVectorIOConfig.sample_run_config(f"~/.llama/distributions/{name}"), + enabled=False, ), Provider( - provider_id="${env.ENABLE_MILVUS:=__disabled__}", + provider_id="milvus", provider_type="inline::milvus", config=MilvusVectorIOConfig.sample_run_config(f"~/.llama/distributions/{name}"), + enabled=False, ), Provider( - provider_id="${env.ENABLE_CHROMADB:=__disabled__}", + provider_id="chromadb", provider_type="remote::chromadb", - config=ChromaVectorIOConfig.sample_run_config(url="${env.CHROMADB_URL:=}"), + config=ChromaVectorIOConfig.sample_run_config(), + enabled=False, ), Provider( - provider_id="${env.ENABLE_PGVECTOR:=__disabled__}", + provider_id="pgvector", provider_type="remote::pgvector", - config=PGVectorVectorIOConfig.sample_run_config( - db="${env.PGVECTOR_DB:=}", - user="${env.PGVECTOR_USER:=}", - password="${env.PGVECTOR_PASSWORD:=}", - ), + config=PGVectorVectorIOConfig.sample_run_config(), + enabled=False, ), ] @@ -216,14 +219,14 @@ def get_distribution_template() -> DistributionTemplate: config=LocalfsFilesImplConfig.sample_run_config(f"~/.llama/distributions/{name}"), ) embedding_provider = Provider( - provider_id="${env.ENABLE_SENTENCE_TRANSFORMERS:=sentence-transformers}", + provider_id="sentence-transformers", provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) post_training_provider = Provider( provider_id="huggingface", provider_type="inline::huggingface", - config=HuggingFacePostTrainingConfig.sample_run_config(f"~/.llama/distributions/{name}"), + config=HuggingFacePostTrainingConfig.sample_run_config(), ) default_tool_groups = [ ToolGroupInput( diff --git a/llama_stack/templates/template.py b/llama_stack/templates/template.py index dceb13c8b..fe4115513 100644 --- a/llama_stack/templates/template.py +++ b/llama_stack/templates/template.py @@ -105,7 +105,8 @@ class RunConfigSettings(BaseModel): if api_providers := self.provider_overrides.get(api_str): # Convert Provider objects to dicts for YAML serialization provider_configs[api_str] = [ - p.model_dump(exclude_none=True) if isinstance(p, Provider) else p for p in api_providers + p.model_dump(exclude_defaults=True, exclude_none=True) if isinstance(p, Provider) else p + for p in api_providers ] continue @@ -133,7 +134,7 @@ class RunConfigSettings(BaseModel): provider_id=provider_id, provider_type=provider_type, config=config, - ).model_dump(exclude_none=True) + ).model_dump(exclude_defaults=True, exclude_none=True) ) # Get unique set of APIs from providers