feat: add static embedding metadata to dynamic model listings for providers using OpenAIMixin (#3547)

# What does this PR do?

- remove auto-download of ollama embedding models
- add embedding model metadata to dynamic listing w/ unit test
- add support and tests for allowed_models
- removed inference provider models.py files where dynamic listing is
enabled
- store embedding metadata in embedding_model_metadata field on
inference providers
- make model_entries optional on ModelRegistryHelper and
LiteLLMOpenAIMixin
- make OpenAIMixin a ModelRegistryHelper
- skip base64 embedding test for remote::ollama, always returns floats
- only use OpenAI client for ollama model listing
- remove unused build_model_entry function
- remove unused get_huggingface_repo function


## Test Plan

ci w/ new tests

llama_stack/providers/remote/inference/anthropic/anthropic.py

@@ -8,14 +8,24 @@ from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOp
 from llama_stack.providers.utils.inference.openai_mixin import OpenAIMixin
 
 from .config import AnthropicConfig
-from .models import MODEL_ENTRIES
 
 
 class AnthropicInferenceAdapter(OpenAIMixin, LiteLLMOpenAIMixin):
+    # source: https://docs.claude.com/en/docs/build-with-claude/embeddings
+    # TODO: add support for voyageai, which is where these models are hosted
+    # embedding_model_metadata = {
+    #     "voyage-3-large": {"embedding_dimension": 1024, "context_length": 32000},  # supports dimensions 256, 512, 1024, 2048
+    #     "voyage-3.5": {"embedding_dimension": 1024, "context_length": 32000},  # supports dimensions 256, 512, 1024, 2048
+    #     "voyage-3.5-lite": {"embedding_dimension": 1024, "context_length": 32000},  # supports dimensions 256, 512, 1024, 2048
+    #     "voyage-code-3": {"embedding_dimension": 1024, "context_length": 32000},  # supports dimensions 256, 512, 1024, 2048
+    #     "voyage-finance-2": {"embedding_dimension": 1024, "context_length": 32000},
+    #     "voyage-law-2": {"embedding_dimension": 1024, "context_length": 16000},
+    #     "voyage-multimodal-3": {"embedding_dimension": 1024, "context_length": 32000},
+    # }
+
     def __init__(self, config: AnthropicConfig) -> None:
         LiteLLMOpenAIMixin.__init__(
             self,
-            MODEL_ENTRIES,
             litellm_provider_name="anthropic",
             api_key_from_config=config.api_key,
             provider_data_api_key_field="anthropic_api_key",

llama_stack/providers/remote/inference/anthropic/models.py

@@ -1,40 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from llama_stack.apis.models import ModelType
-from llama_stack.providers.utils.inference.model_registry import (
-    ProviderModelEntry,
-)
-
-LLM_MODEL_IDS = [
-    "claude-3-5-sonnet-latest",
-    "claude-3-7-sonnet-latest",
-    "claude-3-5-haiku-latest",
-]
-
-SAFETY_MODELS_ENTRIES = []
-
-MODEL_ENTRIES = (
-    [ProviderModelEntry(provider_model_id=m) for m in LLM_MODEL_IDS]
-    + [
-        ProviderModelEntry(
-            provider_model_id="voyage-3",
-            model_type=ModelType.embedding,
-            metadata={"embedding_dimension": 1024, "context_length": 32000},
-        ),
-        ProviderModelEntry(
-            provider_model_id="voyage-3-lite",
-            model_type=ModelType.embedding,
-            metadata={"embedding_dimension": 512, "context_length": 32000},
-        ),
-        ProviderModelEntry(
-            provider_model_id="voyage-code-3",
-            model_type=ModelType.embedding,
-            metadata={"embedding_dimension": 1024, "context_length": 32000},
-        ),
-    ]
-    + SAFETY_MODELS_ENTRIES
-)