refactor(env)!: enhanced environment variable substitution (#2490)

# What does this PR do? This commit significantly improves the environment variable substitution functionality in Llama Stack configuration files: * The version field in configuration files has been changed from string to integer type for better type consistency across build and run configurations. * The environment variable substitution system for ${env.FOO:} was fixed and properly returns an error * The environment variable substitution system for ${env.FOO+} returns None instead of an empty strings, it better matches type annotations in config fields * The system includes automatic type conversion for boolean, integer, and float values. * The error messages have been enhanced to provide clearer guidance when environment variables are missing, including suggestions for using default values or conditional syntax. * Comprehensive documentation has been added to the configuration guide explaining all supported syntax patterns, best practices, and runtime override capabilities. * Multiple provider configurations have been updated to use the new conditional syntax for optional API keys, making the system more flexible for different deployment scenarios. The telemetry configuration has been improved to properly handle optional endpoints with appropriate validation, ensuring that required endpoints are specified when their corresponding sinks are enabled. * There were many instances of ${env.NVIDIA_API_KEY:} that should have caused the code to fail. However, due to a bug, the distro server was still being started, and early validation wasn’t triggered. As a result, failures were likely being handled downstream by the providers. I’ve maintained similar behavior by using ${env.NVIDIA_API_KEY:+}, though I believe this is incorrect for many configurations. I’ll leave it to each provider to correct it as needed. * Environment variable substitution now uses the same syntax as Bash parameter expansion. Signed-off-by: Sébastien Han <seb@redhat.com>
2025-06-27 18:50:41 +00:00 · 2025-06-26 04:50:08 +02:00 · 2025-06-26 04:50:08 +02:00 · 43c1f39bd6
commit 43c1f39bd6
parent 36d70637b9
91 changed files with 1053 additions and 892 deletions
--- a/docs/source/distributions/configuration.md
+++ b/docs/source/distributions/configuration.md
@ -18,7 +18,7 @@ providers:
  - provider_id: ollama
    provider_type: remote::ollama
    config:
-      url: ${env.OLLAMA_URL:http://localhost:11434}
+      url: ${env.OLLAMA_URL:=http://localhost:11434}
  vector_io:
  - provider_id: faiss
    provider_type: inline::faiss
@ -26,7 +26,7 @@ providers:
      kvstore:
        type: sqlite
        namespace: null
-        db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/faiss_store.db
+        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ollama}/faiss_store.db
  safety:
  - provider_id: llama-guard
    provider_type: inline::llama-guard
@ -38,7 +38,7 @@ providers:
      persistence_store:
        type: sqlite
        namespace: null
-        db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/agents_store.db
+        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ollama}/agents_store.db
  telemetry:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
@ -46,7 +46,7 @@ providers:
 metadata_store:
  namespace: null
  type: sqlite
-  db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/registry.db
+  db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ollama}/registry.db
 models:
 - metadata: {}
  model_id: ${env.INFERENCE_MODEL}
@ -85,7 +85,7 @@ providers:
    # config is a dictionary that contains the configuration for the provider.
    # in this case, the configuration is the url of the ollama server
    config:
-      url: ${env.OLLAMA_URL:http://localhost:11434}
+      url: ${env.OLLAMA_URL:=http://localhost:11434}
 ```
 A few things to note:
 - A _provider instance_ is identified with an (id, type, configuration) triplet.
@ -94,6 +94,95 @@ A few things to note:
 - The configuration dictionary is provider-specific.
 - Notice that configuration can reference environment variables (with default values), which are expanded at runtime. When you run a stack server (via docker or via `llama stack run`), you can specify `--env OLLAMA_URL=http://my-server:11434` to override the default value.

+### Environment Variable Substitution
+
+Llama Stack supports environment variable substitution in configuration values using the
+`${env.VARIABLE_NAME}` syntax. This allows you to externalize configuration values and provide
+different settings for different environments. The syntax is inspired by [bash parameter expansion](https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html)
+and follows similar patterns.
+
+#### Basic Syntax
+
+The basic syntax for environment variable substitution is:
+
+```yaml
+config:
+  api_key: ${env.API_KEY}
+  url: ${env.SERVICE_URL}
+```
+
+If the environment variable is not set, the server will raise an error during startup.
+
+#### Default Values
+
+You can provide default values using the `:=` operator:
+
+```yaml
+config:
+  url: ${env.OLLAMA_URL:=http://localhost:11434}
+  port: ${env.PORT:=8321}
+  timeout: ${env.TIMEOUT:=60}
+```
+
+If the environment variable is not set, the default value `http://localhost:11434` will be used.
+Empty defaults are not allowed so `url: ${env.OLLAMA_URL:=}` will raise an error if the environment variable is not set.
+
+#### Conditional Values
+
+You can use the `:+` operator to provide a value only when the environment variable is set:
+
+```yaml
+config:
+  # Only include this field if ENVIRONMENT is set
+  environment: ${env.ENVIRONMENT:+production}
+```
+
+If the environment variable is set, the value after `:+` will be used. If it's not set, the field
+will be omitted with a `None` value.
+So `${env.ENVIRONMENT:+}` is supported, it means that the field will be omitted if the environment
+variable is not set. It can be used to make a field optional and then enabled at runtime when desired.
+
+#### Examples
+
+Here are some common patterns:
+
+```yaml
+# Required environment variable (will error if not set)
+api_key: ${env.OPENAI_API_KEY}
+
+# Optional with default
+base_url: ${env.API_BASE_URL:=https://api.openai.com/v1}
+
+# Conditional field
+debug_mode: ${env.DEBUG:+true}
+
+# Optional field that becomes None if not set
+optional_token: ${env.OPTIONAL_TOKEN:+}
+```
+
+#### Runtime Override
+
+You can override environment variables at runtime when starting the server:
+
+```bash
+# Override specific environment variables
+llama stack run --config run.yaml --env API_KEY=sk-123 --env BASE_URL=https://custom-api.com
+
+# Or set them in your shell
+export API_KEY=sk-123
+export BASE_URL=https://custom-api.com
+llama stack run --config run.yaml
+```
+
+#### Type Safety
+
+The environment variable substitution system is type-safe:
+
+- String values remain strings
+- Empty defaults (`${env.VAR:+}`) are converted to `None` for fields that accept `str | None`
+- Numeric defaults are properly typed (e.g., `${env.PORT:=8321}` becomes an integer)
+- Boolean defaults work correctly (e.g., `${env.DEBUG:=false}` becomes a boolean)
+
 ## Resources

 Finally, let's look at the `models` section:
@ -152,7 +241,7 @@ server:
    config:
      jwks:
        uri: "https://kubernetes.default.svc:8443/openid/v1/jwks"
-        token: "${env.TOKEN:}"
+        token: "${env.TOKEN:+}"
        key_recheck_period: 3600
      tls_cafile: "/path/to/ca.crt"
      issuer: "https://kubernetes.default.svc"
@ -396,12 +485,12 @@ providers:
  - provider_id: vllm-0
    provider_type: remote::vllm
    config:
-      url: ${env.VLLM_URL:http://localhost:8000}
+      url: ${env.VLLM_URL:=http://localhost:8000}
  # this vLLM server serves the llama-guard model (e.g., llama-guard:3b)
  - provider_id: vllm-1
    provider_type: remote::vllm
    config:
-      url: ${env.SAFETY_VLLM_URL:http://localhost:8001}
+      url: ${env.SAFETY_VLLM_URL:=http://localhost:8001}
 ...
 models:
 - metadata: {}
--- a/docs/source/distributions/k8s/stack-configmap.yaml
+++ b/docs/source/distributions/k8s/stack-configmap.yaml
@ -15,10 +15,10 @@ data:
      - provider_id: vllm-inference
        provider_type: remote::vllm
        config:
-          url: ${env.VLLM_URL:http://localhost:8000/v1}
-          max_tokens: ${env.VLLM_MAX_TOKENS:4096}
-          api_token: ${env.VLLM_API_TOKEN:fake}
-          tls_verify: ${env.VLLM_TLS_VERIFY:true}
+          url: ${env.VLLM_URL:=http://localhost:8000/v1}
+          max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+          api_token: ${env.VLLM_API_TOKEN:=fake}
+          tls_verify: ${env.VLLM_TLS_VERIFY:=true}
      - provider_id: vllm-safety
        provider_type: remote::vllm
        config:
@ -30,10 +30,10 @@ data:
        provider_type: inline::sentence-transformers
        config: {}
      vector_io:
-      - provider_id: ${env.ENABLE_CHROMADB+chromadb}
+      - provider_id: ${env.ENABLE_CHROMADB:+chromadb}
        provider_type: remote::chromadb
        config:
-          url: ${env.CHROMADB_URL:}
+          url: ${env.CHROMADB_URL:+}
      safety:
      - provider_id: llama-guard
        provider_type: inline::llama-guard
@ -45,34 +45,34 @@ data:
        config:
          persistence_store:
            type: postgres
-            host: ${env.POSTGRES_HOST:localhost}
-            port: ${env.POSTGRES_PORT:5432}
-            db: ${env.POSTGRES_DB:llamastack}
+            host: ${env.POSTGRES_HOST:=localhost}
+            port: ${env.POSTGRES_PORT:=5432}
+            db: ${env.POSTGRES_DB:=llamastack}
            user: ${env.POSTGRES_USER:llamastack}
-            password: ${env.POSTGRES_PASSWORD:llamastack}
+            password: ${env.POSTGRES_PASSWORD:=llamastack}
          responses_store:
            type: postgres
-            host: ${env.POSTGRES_HOST:localhost}
-            port: ${env.POSTGRES_PORT:5432}
-            db: ${env.POSTGRES_DB:llamastack}
-            user: ${env.POSTGRES_USER:llamastack}
-            password: ${env.POSTGRES_PASSWORD:llamastack}
+            host: ${env.POSTGRES_HOST:=localhost}
+            port: ${env.POSTGRES_PORT:=5432}
+            db: ${env.POSTGRES_DB:=llamastack}
+            user: ${env.POSTGRES_USER:=llamastack}
+            password: ${env.POSTGRES_PASSWORD:=llamastack}
      telemetry:
      - provider_id: meta-reference
        provider_type: inline::meta-reference
        config:
-          service_name: ${env.OTEL_SERVICE_NAME:}
+          service_name: ${env.OTEL_SERVICE_NAME:+}
          sinks: ${env.TELEMETRY_SINKS:console}
      tool_runtime:
      - provider_id: brave-search
        provider_type: remote::brave-search
        config:
-          api_key: ${env.BRAVE_SEARCH_API_KEY:}
+          api_key: ${env.BRAVE_SEARCH_API_KEY:+}
          max_results: 3
      - provider_id: tavily-search
        provider_type: remote::tavily-search
        config:
-          api_key: ${env.TAVILY_SEARCH_API_KEY:}
+          api_key: ${env.TAVILY_SEARCH_API_KEY:+}
          max_results: 3
      - provider_id: rag-runtime
        provider_type: inline::rag-runtime
@ -82,19 +82,19 @@ data:
        config: {}
    metadata_store:
      type: postgres
-      host: ${env.POSTGRES_HOST:localhost}
-      port: ${env.POSTGRES_PORT:5432}
-      db: ${env.POSTGRES_DB:llamastack}
-      user: ${env.POSTGRES_USER:llamastack}
-      password: ${env.POSTGRES_PASSWORD:llamastack}
+      host: ${env.POSTGRES_HOST:=localhost}
+      port: ${env.POSTGRES_PORT:=5432}
+      db: ${env.POSTGRES_DB:=llamastack}
+      user: ${env.POSTGRES_USER:=llamastack}
+      password: ${env.POSTGRES_PASSWORD:=llamastack}
      table_name: llamastack_kvstore
    inference_store:
      type: postgres
-      host: ${env.POSTGRES_HOST:localhost}
-      port: ${env.POSTGRES_PORT:5432}
-      db: ${env.POSTGRES_DB:llamastack}
-      user: ${env.POSTGRES_USER:llamastack}
-      password: ${env.POSTGRES_PASSWORD:llamastack}
+      host: ${env.POSTGRES_HOST:=localhost}
+      port: ${env.POSTGRES_PORT:=5432}
+      db: ${env.POSTGRES_DB:=llamastack}
+      user: ${env.POSTGRES_USER:=llamastack}
+      password: ${env.POSTGRES_PASSWORD:=llamastack}
    models:
    - metadata:
        embedding_dimension: 384
@ -106,11 +106,11 @@ data:
      provider_id: vllm-inference
      model_type: llm
    - metadata: {}
-      model_id: ${env.SAFETY_MODEL:meta-llama/Llama-Guard-3-1B}
+      model_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
      provider_id: vllm-safety
      model_type: llm
    shields:
-    - shield_id: ${env.SAFETY_MODEL:meta-llama/Llama-Guard-3-1B}
+    - shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
    vector_dbs: []
    datasets: []
    scoring_fns: []
--- a/docs/source/distributions/k8s/stack_run_config.yaml
+++ b/docs/source/distributions/k8s/stack_run_config.yaml
@ -12,25 +12,25 @@ providers:
  - provider_id: vllm-inference
    provider_type: remote::vllm
    config:
-      url: ${env.VLLM_URL:http://localhost:8000/v1}
-      max_tokens: ${env.VLLM_MAX_TOKENS:4096}
-      api_token: ${env.VLLM_API_TOKEN:fake}
-      tls_verify: ${env.VLLM_TLS_VERIFY:true}
+      url: ${env.VLLM_URL:=http://localhost:8000/v1}
+      max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+      api_token: ${env.VLLM_API_TOKEN:=fake}
+      tls_verify: ${env.VLLM_TLS_VERIFY:=true}
  - provider_id: vllm-safety
    provider_type: remote::vllm
    config:
-      url: ${env.VLLM_SAFETY_URL:http://localhost:8000/v1}
-      max_tokens: ${env.VLLM_MAX_TOKENS:4096}
-      api_token: ${env.VLLM_API_TOKEN:fake}
-      tls_verify: ${env.VLLM_TLS_VERIFY:true}
+      url: ${env.VLLM_SAFETY_URL:=http://localhost:8000/v1}
+      max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+      api_token: ${env.VLLM_API_TOKEN:=fake}
+      tls_verify: ${env.VLLM_TLS_VERIFY:=true}
  - provider_id: sentence-transformers
    provider_type: inline::sentence-transformers
    config: {}
  vector_io:
-  - provider_id: ${env.ENABLE_CHROMADB+chromadb}
+  - provider_id: ${env.ENABLE_CHROMADB:+chromadb}
    provider_type: remote::chromadb
    config:
-      url: ${env.CHROMADB_URL:}
+      url: ${env.CHROMADB_URL:+}
  safety:
  - provider_id: llama-guard
    provider_type: inline::llama-guard
@ -42,34 +42,34 @@ providers:
    config:
      persistence_store:
        type: postgres
-        host: ${env.POSTGRES_HOST:localhost}
-        port: ${env.POSTGRES_PORT:5432}
-        db: ${env.POSTGRES_DB:llamastack}
-        user: ${env.POSTGRES_USER:llamastack}
-        password: ${env.POSTGRES_PASSWORD:llamastack}
+        host: ${env.POSTGRES_HOST:=localhost}
+        port: ${env.POSTGRES_PORT:=5432}
+        db: ${env.POSTGRES_DB:=llamastack}
+        user: ${env.POSTGRES_USER:=llamastack}
+        password: ${env.POSTGRES_PASSWORD:=llamastack}
      responses_store:
        type: postgres
-        host: ${env.POSTGRES_HOST:localhost}
-        port: ${env.POSTGRES_PORT:5432}
-        db: ${env.POSTGRES_DB:llamastack}
-        user: ${env.POSTGRES_USER:llamastack}
-        password: ${env.POSTGRES_PASSWORD:llamastack}
+        host: ${env.POSTGRES_HOST:=localhost}
+        port: ${env.POSTGRES_PORT:=5432}
+        db: ${env.POSTGRES_DB:=llamastack}
+        user: ${env.POSTGRES_USER:=llamastack}
+        password: ${env.POSTGRES_PASSWORD:=llamastack}
  telemetry:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config:
-      service_name: ${env.OTEL_SERVICE_NAME:}
-      sinks: ${env.TELEMETRY_SINKS:console}
+      service_name: ${env.OTEL_SERVICE_NAME:+console}
+      sinks: ${env.TELEMETRY_SINKS:+console}
  tool_runtime:
  - provider_id: brave-search
    provider_type: remote::brave-search
    config:
-      api_key: ${env.BRAVE_SEARCH_API_KEY:}
+      api_key: ${env.BRAVE_SEARCH_API_KEY:+}
      max_results: 3
  - provider_id: tavily-search
    provider_type: remote::tavily-search
    config:
-      api_key: ${env.TAVILY_SEARCH_API_KEY:}
+      api_key: ${env.TAVILY_SEARCH_API_KEY:+}
      max_results: 3
  - provider_id: rag-runtime
    provider_type: inline::rag-runtime
@ -79,19 +79,19 @@ providers:
    config: {}
 metadata_store:
  type: postgres
-  host: ${env.POSTGRES_HOST:localhost}
-  port: ${env.POSTGRES_PORT:5432}
-  db: ${env.POSTGRES_DB:llamastack}
-  user: ${env.POSTGRES_USER:llamastack}
-  password: ${env.POSTGRES_PASSWORD:llamastack}
+  host: ${env.POSTGRES_HOST:=localhost}
+  port: ${env.POSTGRES_PORT:=5432}
+  db: ${env.POSTGRES_DB:=llamastack}
+  user: ${env.POSTGRES_USER:=llamastack}
+  password: ${env.POSTGRES_PASSWORD:=llamastack}
  table_name: llamastack_kvstore
 inference_store:
  type: postgres
-  host: ${env.POSTGRES_HOST:localhost}
-  port: ${env.POSTGRES_PORT:5432}
-  db: ${env.POSTGRES_DB:llamastack}
-  user: ${env.POSTGRES_USER:llamastack}
-  password: ${env.POSTGRES_PASSWORD:llamastack}
+  host: ${env.POSTGRES_HOST:=localhost}
+  port: ${env.POSTGRES_PORT:=5432}
+  db: ${env.POSTGRES_DB:=llamastack}
+  user: ${env.POSTGRES_USER:=llamastack}
+  password: ${env.POSTGRES_PASSWORD:=llamastack}
 models:
 - metadata:
    embedding_dimension: 384
@ -103,11 +103,11 @@ models:
  provider_id: vllm-inference
  model_type: llm
 - metadata: {}
-  model_id: ${env.SAFETY_MODEL:meta-llama/Llama-Guard-3-1B}
+  model_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
  provider_id: vllm-safety
  model_type: llm
 shields:
- shield_id: ${env.SAFETY_MODEL:meta-llama/Llama-Guard-3-1B}
+- shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
 vector_dbs: []
 datasets: []
 scoring_fns: []