feat: allow for provider immutability

being able to update providers means that admins should have the ability to turn this feature off. introduce `immutable` as a field in the Provider class. Defauling to false means all providers can be updated by default, but an admin at runtime can choose to set this to True to disable provider updating

Signed-off-by: Charlie Doern <cdoern@redhat.com>

llama_stack/templates/open-benchmark/run.yaml

@@ -16,33 +16,40 @@ providers:
     provider_type: remote::openai
     config:
       api_key: ${env.OPENAI_API_KEY:=}
+    immutable: false
   - provider_id: anthropic
     provider_type: remote::anthropic
     config:
       api_key: ${env.ANTHROPIC_API_KEY:=}
+    immutable: false
   - provider_id: gemini
     provider_type: remote::gemini
     config:
       api_key: ${env.GEMINI_API_KEY:=}
+    immutable: false
   - provider_id: groq
     provider_type: remote::groq
     config:
       url: https://api.groq.com
       api_key: ${env.GROQ_API_KEY:=}
+    immutable: false
   - provider_id: together
     provider_type: remote::together
     config:
       url: https://api.together.xyz/v1
       api_key: ${env.TOGETHER_API_KEY:=}
+    immutable: false
   vector_io:
   - provider_id: sqlite-vec
     provider_type: inline::sqlite-vec
     config:
       db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/sqlite_vec.db
+    immutable: false
   - provider_id: ${env.ENABLE_CHROMADB:+chromadb}
     provider_type: remote::chromadb
     config:
       url: ${env.CHROMADB_URL:=}
+    immutable: false
   - provider_id: ${env.ENABLE_PGVECTOR:+pgvector}
     provider_type: remote::pgvector
     config:
@@ -51,11 +58,13 @@ providers:
       db: ${env.PGVECTOR_DB:=}
       user: ${env.PGVECTOR_USER:=}
       password: ${env.PGVECTOR_PASSWORD:=}
+    immutable: false
   safety:
   - provider_id: llama-guard
     provider_type: inline::llama-guard
     config:
       excluded_categories: []
+    immutable: false
   agents:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
@@ -67,6 +76,7 @@ providers:
       responses_store:
         type: sqlite
         db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/responses_store.db
+    immutable: false
   telemetry:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
@@ -74,6 +84,7 @@ providers:
       service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
       sinks: ${env.TELEMETRY_SINKS:=console,sqlite}
       sqlite_db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/trace_store.db
+    immutable: false
   eval:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
@@ -82,6 +93,7 @@ providers:
         type: sqlite
         namespace: null
         db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/meta_reference_eval.db
+    immutable: false
   datasetio:
   - provider_id: huggingface
     provider_type: remote::huggingface
@@ -90,6 +102,7 @@ providers:
         type: sqlite
         namespace: null
         db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/huggingface_datasetio.db
+    immutable: false
   - provider_id: localfs
     provider_type: inline::localfs
     config:
@@ -97,34 +110,42 @@ providers:
         type: sqlite
         namespace: null
         db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/localfs_datasetio.db
+    immutable: false
   scoring:
   - provider_id: basic
     provider_type: inline::basic
     config: {}
+    immutable: false
   - provider_id: llm-as-judge
     provider_type: inline::llm-as-judge
     config: {}
+    immutable: false
   - provider_id: braintrust
     provider_type: inline::braintrust
     config:
       openai_api_key: ${env.OPENAI_API_KEY:=}
+    immutable: false
   tool_runtime:
   - provider_id: brave-search
     provider_type: remote::brave-search
     config:
       api_key: ${env.BRAVE_SEARCH_API_KEY:=}
       max_results: 3
+    immutable: false
   - provider_id: tavily-search
     provider_type: remote::tavily-search
     config:
       api_key: ${env.TAVILY_SEARCH_API_KEY:=}
       max_results: 3
+    immutable: false
   - provider_id: rag-runtime
     provider_type: inline::rag-runtime
     config: {}
+    immutable: false
   - provider_id: model-context-protocol
     provider_type: remote::model-context-protocol
     config: {}
+    immutable: false
 metadata_store:
   type: sqlite
   db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/registry.db