From 8e1a2b47036f26333558459b1c94e4baa4008f16 Mon Sep 17 00:00:00 2001 From: ehhuang Date: Tue, 22 Jul 2025 10:25:36 -0700 Subject: [PATCH] chore: remove *_openai_compat providers (#2849) # What does this PR do? These are no longer needed as llama-stack-evals can run against OAI endpoints directly. ## Test Plan --- docs/source/providers/inference/index.md | 5 -- llama_stack/providers/registry/inference.py | 55 ------------------- .../cerebras_openai_compat/__init__.py | 17 ------ .../cerebras_openai_compat/cerebras.py | 30 ---------- .../cerebras_openai_compat/config.py | 38 ------------- .../fireworks_openai_compat/__init__.py | 17 ------ .../fireworks_openai_compat/config.py | 38 ------------- .../fireworks_openai_compat/fireworks.py | 30 ---------- .../inference/groq_openai_compat/__init__.py | 17 ------ .../inference/groq_openai_compat/config.py | 38 ------------- .../inference/groq_openai_compat/groq.py | 30 ---------- .../sambanova_openai_compat/__init__.py | 17 ------ .../sambanova_openai_compat/config.py | 38 ------------- .../sambanova_openai_compat/sambanova.py | 30 ---------- .../together_openai_compat/__init__.py | 17 ------ .../together_openai_compat/config.py | 38 ------------- .../together_openai_compat/together.py | 30 ---------- llama_stack/templates/ci-tests/build.yaml | 5 -- llama_stack/templates/ci-tests/run.yaml | 25 --------- llama_stack/templates/starter/build.yaml | 5 -- llama_stack/templates/starter/run.yaml | 25 --------- 21 files changed, 545 deletions(-) delete mode 100644 llama_stack/providers/remote/inference/cerebras_openai_compat/__init__.py delete mode 100644 llama_stack/providers/remote/inference/cerebras_openai_compat/cerebras.py delete mode 100644 llama_stack/providers/remote/inference/cerebras_openai_compat/config.py delete mode 100644 llama_stack/providers/remote/inference/fireworks_openai_compat/__init__.py delete mode 100644 llama_stack/providers/remote/inference/fireworks_openai_compat/config.py delete mode 100644 llama_stack/providers/remote/inference/fireworks_openai_compat/fireworks.py delete mode 100644 llama_stack/providers/remote/inference/groq_openai_compat/__init__.py delete mode 100644 llama_stack/providers/remote/inference/groq_openai_compat/config.py delete mode 100644 llama_stack/providers/remote/inference/groq_openai_compat/groq.py delete mode 100644 llama_stack/providers/remote/inference/sambanova_openai_compat/__init__.py delete mode 100644 llama_stack/providers/remote/inference/sambanova_openai_compat/config.py delete mode 100644 llama_stack/providers/remote/inference/sambanova_openai_compat/sambanova.py delete mode 100644 llama_stack/providers/remote/inference/together_openai_compat/__init__.py delete mode 100644 llama_stack/providers/remote/inference/together_openai_compat/config.py delete mode 100644 llama_stack/providers/remote/inference/together_openai_compat/together.py diff --git a/docs/source/providers/inference/index.md b/docs/source/providers/inference/index.md index 6582e08de..dcc6da5b5 100644 --- a/docs/source/providers/inference/index.md +++ b/docs/source/providers/inference/index.md @@ -7,13 +7,10 @@ This section contains documentation for all available providers for the **infere - [remote::anthropic](remote_anthropic.md) - [remote::bedrock](remote_bedrock.md) - [remote::cerebras](remote_cerebras.md) -- [remote::cerebras-openai-compat](remote_cerebras-openai-compat.md) - [remote::databricks](remote_databricks.md) - [remote::fireworks](remote_fireworks.md) -- [remote::fireworks-openai-compat](remote_fireworks-openai-compat.md) - [remote::gemini](remote_gemini.md) - [remote::groq](remote_groq.md) -- [remote::groq-openai-compat](remote_groq-openai-compat.md) - [remote::hf::endpoint](remote_hf_endpoint.md) - [remote::hf::serverless](remote_hf_serverless.md) - [remote::llama-openai-compat](remote_llama-openai-compat.md) @@ -23,9 +20,7 @@ This section contains documentation for all available providers for the **infere - [remote::passthrough](remote_passthrough.md) - [remote::runpod](remote_runpod.md) - [remote::sambanova](remote_sambanova.md) -- [remote::sambanova-openai-compat](remote_sambanova-openai-compat.md) - [remote::tgi](remote_tgi.md) - [remote::together](remote_together.md) -- [remote::together-openai-compat](remote_together-openai-compat.md) - [remote::vllm](remote_vllm.md) - [remote::watsonx](remote_watsonx.md) \ No newline at end of file diff --git a/llama_stack/providers/registry/inference.py b/llama_stack/providers/registry/inference.py index ffd30a5b5..a8bc96a77 100644 --- a/llama_stack/providers/registry/inference.py +++ b/llama_stack/providers/registry/inference.py @@ -224,17 +224,6 @@ def available_providers() -> list[ProviderSpec]: description="Groq inference provider for ultra-fast inference using Groq's LPU technology.", ), ), - remote_provider_spec( - api=Api.inference, - adapter=AdapterSpec( - adapter_type="fireworks-openai-compat", - pip_packages=["litellm"], - module="llama_stack.providers.remote.inference.fireworks_openai_compat", - config_class="llama_stack.providers.remote.inference.fireworks_openai_compat.config.FireworksCompatConfig", - provider_data_validator="llama_stack.providers.remote.inference.fireworks_openai_compat.config.FireworksProviderDataValidator", - description="Fireworks AI OpenAI-compatible provider for using Fireworks models with OpenAI API format.", - ), - ), remote_provider_spec( api=Api.inference, adapter=AdapterSpec( @@ -246,50 +235,6 @@ def available_providers() -> list[ProviderSpec]: description="Llama OpenAI-compatible provider for using Llama models with OpenAI API format.", ), ), - remote_provider_spec( - api=Api.inference, - adapter=AdapterSpec( - adapter_type="together-openai-compat", - pip_packages=["litellm"], - module="llama_stack.providers.remote.inference.together_openai_compat", - config_class="llama_stack.providers.remote.inference.together_openai_compat.config.TogetherCompatConfig", - provider_data_validator="llama_stack.providers.remote.inference.together_openai_compat.config.TogetherProviderDataValidator", - description="Together AI OpenAI-compatible provider for using Together models with OpenAI API format.", - ), - ), - remote_provider_spec( - api=Api.inference, - adapter=AdapterSpec( - adapter_type="groq-openai-compat", - pip_packages=["litellm"], - module="llama_stack.providers.remote.inference.groq_openai_compat", - config_class="llama_stack.providers.remote.inference.groq_openai_compat.config.GroqCompatConfig", - provider_data_validator="llama_stack.providers.remote.inference.groq_openai_compat.config.GroqProviderDataValidator", - description="Groq OpenAI-compatible provider for using Groq models with OpenAI API format.", - ), - ), - remote_provider_spec( - api=Api.inference, - adapter=AdapterSpec( - adapter_type="sambanova-openai-compat", - pip_packages=["litellm"], - module="llama_stack.providers.remote.inference.sambanova_openai_compat", - config_class="llama_stack.providers.remote.inference.sambanova_openai_compat.config.SambaNovaCompatConfig", - provider_data_validator="llama_stack.providers.remote.inference.sambanova_openai_compat.config.SambaNovaProviderDataValidator", - description="SambaNova OpenAI-compatible provider for using SambaNova models with OpenAI API format.", - ), - ), - remote_provider_spec( - api=Api.inference, - adapter=AdapterSpec( - adapter_type="cerebras-openai-compat", - pip_packages=["litellm"], - module="llama_stack.providers.remote.inference.cerebras_openai_compat", - config_class="llama_stack.providers.remote.inference.cerebras_openai_compat.config.CerebrasCompatConfig", - provider_data_validator="llama_stack.providers.remote.inference.cerebras_openai_compat.config.CerebrasProviderDataValidator", - description="Cerebras OpenAI-compatible provider for using Cerebras models with OpenAI API format.", - ), - ), remote_provider_spec( api=Api.inference, adapter=AdapterSpec( diff --git a/llama_stack/providers/remote/inference/cerebras_openai_compat/__init__.py b/llama_stack/providers/remote/inference/cerebras_openai_compat/__init__.py deleted file mode 100644 index 523a8dfe7..000000000 --- a/llama_stack/providers/remote/inference/cerebras_openai_compat/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.apis.inference import InferenceProvider - -from .config import CerebrasCompatConfig - - -async def get_adapter_impl(config: CerebrasCompatConfig, _deps) -> InferenceProvider: - # import dynamically so the import is used only when it is needed - from .cerebras import CerebrasCompatInferenceAdapter - - adapter = CerebrasCompatInferenceAdapter(config) - return adapter diff --git a/llama_stack/providers/remote/inference/cerebras_openai_compat/cerebras.py b/llama_stack/providers/remote/inference/cerebras_openai_compat/cerebras.py deleted file mode 100644 index b3f109dcc..000000000 --- a/llama_stack/providers/remote/inference/cerebras_openai_compat/cerebras.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.providers.remote.inference.cerebras_openai_compat.config import CerebrasCompatConfig -from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOpenAIMixin - -from ..cerebras.models import MODEL_ENTRIES - - -class CerebrasCompatInferenceAdapter(LiteLLMOpenAIMixin): - _config: CerebrasCompatConfig - - def __init__(self, config: CerebrasCompatConfig): - LiteLLMOpenAIMixin.__init__( - self, - model_entries=MODEL_ENTRIES, - api_key_from_config=config.api_key, - provider_data_api_key_field="cerebras_api_key", - openai_compat_api_base=config.openai_compat_api_base, - ) - self.config = config - - async def initialize(self): - await super().initialize() - - async def shutdown(self): - await super().shutdown() diff --git a/llama_stack/providers/remote/inference/cerebras_openai_compat/config.py b/llama_stack/providers/remote/inference/cerebras_openai_compat/config.py deleted file mode 100644 index cb8daff6a..000000000 --- a/llama_stack/providers/remote/inference/cerebras_openai_compat/config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Any - -from pydantic import BaseModel, Field - -from llama_stack.schema_utils import json_schema_type - - -class CerebrasProviderDataValidator(BaseModel): - cerebras_api_key: str | None = Field( - default=None, - description="API key for Cerebras models", - ) - - -@json_schema_type -class CerebrasCompatConfig(BaseModel): - api_key: str | None = Field( - default=None, - description="The Cerebras API key", - ) - - openai_compat_api_base: str = Field( - default="https://api.cerebras.ai/v1", - description="The URL for the Cerebras API server", - ) - - @classmethod - def sample_run_config(cls, api_key: str = "${env.CEREBRAS_API_KEY}", **kwargs) -> dict[str, Any]: - return { - "openai_compat_api_base": "https://api.cerebras.ai/v1", - "api_key": api_key, - } diff --git a/llama_stack/providers/remote/inference/fireworks_openai_compat/__init__.py b/llama_stack/providers/remote/inference/fireworks_openai_compat/__init__.py deleted file mode 100644 index 15a666cb6..000000000 --- a/llama_stack/providers/remote/inference/fireworks_openai_compat/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.apis.inference import InferenceProvider - -from .config import FireworksCompatConfig - - -async def get_adapter_impl(config: FireworksCompatConfig, _deps) -> InferenceProvider: - # import dynamically so the import is used only when it is needed - from .fireworks import FireworksCompatInferenceAdapter - - adapter = FireworksCompatInferenceAdapter(config) - return adapter diff --git a/llama_stack/providers/remote/inference/fireworks_openai_compat/config.py b/llama_stack/providers/remote/inference/fireworks_openai_compat/config.py deleted file mode 100644 index bf38cdd2b..000000000 --- a/llama_stack/providers/remote/inference/fireworks_openai_compat/config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Any - -from pydantic import BaseModel, Field - -from llama_stack.schema_utils import json_schema_type - - -class FireworksProviderDataValidator(BaseModel): - fireworks_api_key: str | None = Field( - default=None, - description="API key for Fireworks models", - ) - - -@json_schema_type -class FireworksCompatConfig(BaseModel): - api_key: str | None = Field( - default=None, - description="The Fireworks API key", - ) - - openai_compat_api_base: str = Field( - default="https://api.fireworks.ai/inference/v1", - description="The URL for the Fireworks API server", - ) - - @classmethod - def sample_run_config(cls, api_key: str = "${env.FIREWORKS_API_KEY}", **kwargs) -> dict[str, Any]: - return { - "openai_compat_api_base": "https://api.fireworks.ai/inference/v1", - "api_key": api_key, - } diff --git a/llama_stack/providers/remote/inference/fireworks_openai_compat/fireworks.py b/llama_stack/providers/remote/inference/fireworks_openai_compat/fireworks.py deleted file mode 100644 index f6045e0eb..000000000 --- a/llama_stack/providers/remote/inference/fireworks_openai_compat/fireworks.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.providers.remote.inference.fireworks_openai_compat.config import FireworksCompatConfig -from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOpenAIMixin - -from ..fireworks.models import MODEL_ENTRIES - - -class FireworksCompatInferenceAdapter(LiteLLMOpenAIMixin): - _config: FireworksCompatConfig - - def __init__(self, config: FireworksCompatConfig): - LiteLLMOpenAIMixin.__init__( - self, - model_entries=MODEL_ENTRIES, - api_key_from_config=config.api_key, - provider_data_api_key_field="fireworks_api_key", - openai_compat_api_base=config.openai_compat_api_base, - ) - self.config = config - - async def initialize(self): - await super().initialize() - - async def shutdown(self): - await super().shutdown() diff --git a/llama_stack/providers/remote/inference/groq_openai_compat/__init__.py b/llama_stack/providers/remote/inference/groq_openai_compat/__init__.py deleted file mode 100644 index 794cdebd7..000000000 --- a/llama_stack/providers/remote/inference/groq_openai_compat/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.apis.inference import InferenceProvider - -from .config import GroqCompatConfig - - -async def get_adapter_impl(config: GroqCompatConfig, _deps) -> InferenceProvider: - # import dynamically so the import is used only when it is needed - from .groq import GroqCompatInferenceAdapter - - adapter = GroqCompatInferenceAdapter(config) - return adapter diff --git a/llama_stack/providers/remote/inference/groq_openai_compat/config.py b/llama_stack/providers/remote/inference/groq_openai_compat/config.py deleted file mode 100644 index 481f740f9..000000000 --- a/llama_stack/providers/remote/inference/groq_openai_compat/config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Any - -from pydantic import BaseModel, Field - -from llama_stack.schema_utils import json_schema_type - - -class GroqProviderDataValidator(BaseModel): - groq_api_key: str | None = Field( - default=None, - description="API key for Groq models", - ) - - -@json_schema_type -class GroqCompatConfig(BaseModel): - api_key: str | None = Field( - default=None, - description="The Groq API key", - ) - - openai_compat_api_base: str = Field( - default="https://api.groq.com/openai/v1", - description="The URL for the Groq API server", - ) - - @classmethod - def sample_run_config(cls, api_key: str = "${env.GROQ_API_KEY}", **kwargs) -> dict[str, Any]: - return { - "openai_compat_api_base": "https://api.groq.com/openai/v1", - "api_key": api_key, - } diff --git a/llama_stack/providers/remote/inference/groq_openai_compat/groq.py b/llama_stack/providers/remote/inference/groq_openai_compat/groq.py deleted file mode 100644 index 30e18cd06..000000000 --- a/llama_stack/providers/remote/inference/groq_openai_compat/groq.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.providers.remote.inference.groq_openai_compat.config import GroqCompatConfig -from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOpenAIMixin - -from ..groq.models import MODEL_ENTRIES - - -class GroqCompatInferenceAdapter(LiteLLMOpenAIMixin): - _config: GroqCompatConfig - - def __init__(self, config: GroqCompatConfig): - LiteLLMOpenAIMixin.__init__( - self, - model_entries=MODEL_ENTRIES, - api_key_from_config=config.api_key, - provider_data_api_key_field="groq_api_key", - openai_compat_api_base=config.openai_compat_api_base, - ) - self.config = config - - async def initialize(self): - await super().initialize() - - async def shutdown(self): - await super().shutdown() diff --git a/llama_stack/providers/remote/inference/sambanova_openai_compat/__init__.py b/llama_stack/providers/remote/inference/sambanova_openai_compat/__init__.py deleted file mode 100644 index 60afe91ca..000000000 --- a/llama_stack/providers/remote/inference/sambanova_openai_compat/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.apis.inference import InferenceProvider - -from .config import SambaNovaCompatConfig - - -async def get_adapter_impl(config: SambaNovaCompatConfig, _deps) -> InferenceProvider: - # import dynamically so the import is used only when it is needed - from .sambanova import SambaNovaCompatInferenceAdapter - - adapter = SambaNovaCompatInferenceAdapter(config) - return adapter diff --git a/llama_stack/providers/remote/inference/sambanova_openai_compat/config.py b/llama_stack/providers/remote/inference/sambanova_openai_compat/config.py deleted file mode 100644 index 072fa85d1..000000000 --- a/llama_stack/providers/remote/inference/sambanova_openai_compat/config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Any - -from pydantic import BaseModel, Field - -from llama_stack.schema_utils import json_schema_type - - -class SambaNovaProviderDataValidator(BaseModel): - sambanova_api_key: str | None = Field( - default=None, - description="API key for SambaNova models", - ) - - -@json_schema_type -class SambaNovaCompatConfig(BaseModel): - api_key: str | None = Field( - default=None, - description="The SambaNova API key", - ) - - openai_compat_api_base: str = Field( - default="https://api.sambanova.ai/v1", - description="The URL for the SambaNova API server", - ) - - @classmethod - def sample_run_config(cls, api_key: str = "${env.SAMBANOVA_API_KEY}", **kwargs) -> dict[str, Any]: - return { - "openai_compat_api_base": "https://api.sambanova.ai/v1", - "api_key": api_key, - } diff --git a/llama_stack/providers/remote/inference/sambanova_openai_compat/sambanova.py b/llama_stack/providers/remote/inference/sambanova_openai_compat/sambanova.py deleted file mode 100644 index aa59028b6..000000000 --- a/llama_stack/providers/remote/inference/sambanova_openai_compat/sambanova.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.providers.remote.inference.sambanova_openai_compat.config import SambaNovaCompatConfig -from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOpenAIMixin - -from ..sambanova.models import MODEL_ENTRIES - - -class SambaNovaCompatInferenceAdapter(LiteLLMOpenAIMixin): - _config: SambaNovaCompatConfig - - def __init__(self, config: SambaNovaCompatConfig): - LiteLLMOpenAIMixin.__init__( - self, - model_entries=MODEL_ENTRIES, - api_key_from_config=config.api_key, - provider_data_api_key_field="sambanova_api_key", - openai_compat_api_base=config.openai_compat_api_base, - ) - self.config = config - - async def initialize(self): - await super().initialize() - - async def shutdown(self): - await super().shutdown() diff --git a/llama_stack/providers/remote/inference/together_openai_compat/__init__.py b/llama_stack/providers/remote/inference/together_openai_compat/__init__.py deleted file mode 100644 index 8213fc5f4..000000000 --- a/llama_stack/providers/remote/inference/together_openai_compat/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.apis.inference import InferenceProvider - -from .config import TogetherCompatConfig - - -async def get_adapter_impl(config: TogetherCompatConfig, _deps) -> InferenceProvider: - # import dynamically so the import is used only when it is needed - from .together import TogetherCompatInferenceAdapter - - adapter = TogetherCompatInferenceAdapter(config) - return adapter diff --git a/llama_stack/providers/remote/inference/together_openai_compat/config.py b/llama_stack/providers/remote/inference/together_openai_compat/config.py deleted file mode 100644 index 0c6d4f748..000000000 --- a/llama_stack/providers/remote/inference/together_openai_compat/config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Any - -from pydantic import BaseModel, Field - -from llama_stack.schema_utils import json_schema_type - - -class TogetherProviderDataValidator(BaseModel): - together_api_key: str | None = Field( - default=None, - description="API key for Together models", - ) - - -@json_schema_type -class TogetherCompatConfig(BaseModel): - api_key: str | None = Field( - default=None, - description="The Together API key", - ) - - openai_compat_api_base: str = Field( - default="https://api.together.xyz/v1", - description="The URL for the Together API server", - ) - - @classmethod - def sample_run_config(cls, api_key: str = "${env.TOGETHER_API_KEY}", **kwargs) -> dict[str, Any]: - return { - "openai_compat_api_base": "https://api.together.xyz/v1", - "api_key": api_key, - } diff --git a/llama_stack/providers/remote/inference/together_openai_compat/together.py b/llama_stack/providers/remote/inference/together_openai_compat/together.py deleted file mode 100644 index b463f5c35..000000000 --- a/llama_stack/providers/remote/inference/together_openai_compat/together.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack.providers.remote.inference.together_openai_compat.config import TogetherCompatConfig -from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOpenAIMixin - -from ..together.models import MODEL_ENTRIES - - -class TogetherCompatInferenceAdapter(LiteLLMOpenAIMixin): - _config: TogetherCompatConfig - - def __init__(self, config: TogetherCompatConfig): - LiteLLMOpenAIMixin.__init__( - self, - model_entries=MODEL_ENTRIES, - api_key_from_config=config.api_key, - provider_data_api_key_field="together_api_key", - openai_compat_api_base=config.openai_compat_api_base, - ) - self.config = config - - async def initialize(self): - await super().initialize() - - async def shutdown(self): - await super().shutdown() diff --git a/llama_stack/templates/ci-tests/build.yaml b/llama_stack/templates/ci-tests/build.yaml index 0aed1d185..625e36e4f 100644 --- a/llama_stack/templates/ci-tests/build.yaml +++ b/llama_stack/templates/ci-tests/build.yaml @@ -19,12 +19,7 @@ distribution_spec: - remote::anthropic - remote::gemini - remote::groq - - remote::fireworks-openai-compat - remote::llama-openai-compat - - remote::together-openai-compat - - remote::groq-openai-compat - - remote::sambanova-openai-compat - - remote::cerebras-openai-compat - remote::sambanova - remote::passthrough - inline::sentence-transformers diff --git a/llama_stack/templates/ci-tests/run.yaml b/llama_stack/templates/ci-tests/run.yaml index cc7378c97..3757c6e60 100644 --- a/llama_stack/templates/ci-tests/run.yaml +++ b/llama_stack/templates/ci-tests/run.yaml @@ -90,36 +90,11 @@ providers: config: url: https://api.groq.com api_key: ${env.GROQ_API_KEY} - - provider_id: ${env.ENABLE_FIREWORKS_OPENAI_COMPAT:=__disabled__} - provider_type: remote::fireworks-openai-compat - config: - openai_compat_api_base: https://api.fireworks.ai/inference/v1 - api_key: ${env.FIREWORKS_API_KEY} - provider_id: ${env.ENABLE_LLAMA_OPENAI_COMPAT:=__disabled__} provider_type: remote::llama-openai-compat config: openai_compat_api_base: https://api.llama.com/compat/v1/ api_key: ${env.LLAMA_API_KEY} - - provider_id: ${env.ENABLE_TOGETHER_OPENAI_COMPAT:=__disabled__} - provider_type: remote::together-openai-compat - config: - openai_compat_api_base: https://api.together.xyz/v1 - api_key: ${env.TOGETHER_API_KEY} - - provider_id: ${env.ENABLE_GROQ_OPENAI_COMPAT:=__disabled__} - provider_type: remote::groq-openai-compat - config: - openai_compat_api_base: https://api.groq.com/openai/v1 - api_key: ${env.GROQ_API_KEY} - - provider_id: ${env.ENABLE_SAMBANOVA_OPENAI_COMPAT:=__disabled__} - provider_type: remote::sambanova-openai-compat - config: - openai_compat_api_base: https://api.sambanova.ai/v1 - api_key: ${env.SAMBANOVA_API_KEY} - - provider_id: ${env.ENABLE_CEREBRAS_OPENAI_COMPAT:=__disabled__} - provider_type: remote::cerebras-openai-compat - config: - openai_compat_api_base: https://api.cerebras.ai/v1 - api_key: ${env.CEREBRAS_API_KEY} - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} provider_type: remote::sambanova config: diff --git a/llama_stack/templates/starter/build.yaml b/llama_stack/templates/starter/build.yaml index dc7565d46..8180124f6 100644 --- a/llama_stack/templates/starter/build.yaml +++ b/llama_stack/templates/starter/build.yaml @@ -19,12 +19,7 @@ distribution_spec: - remote::anthropic - remote::gemini - remote::groq - - remote::fireworks-openai-compat - remote::llama-openai-compat - - remote::together-openai-compat - - remote::groq-openai-compat - - remote::sambanova-openai-compat - - remote::cerebras-openai-compat - remote::sambanova - remote::passthrough - inline::sentence-transformers diff --git a/llama_stack/templates/starter/run.yaml b/llama_stack/templates/starter/run.yaml index 46573848c..62e96d3b5 100644 --- a/llama_stack/templates/starter/run.yaml +++ b/llama_stack/templates/starter/run.yaml @@ -90,36 +90,11 @@ providers: config: url: https://api.groq.com api_key: ${env.GROQ_API_KEY} - - provider_id: ${env.ENABLE_FIREWORKS_OPENAI_COMPAT:=__disabled__} - provider_type: remote::fireworks-openai-compat - config: - openai_compat_api_base: https://api.fireworks.ai/inference/v1 - api_key: ${env.FIREWORKS_API_KEY} - provider_id: ${env.ENABLE_LLAMA_OPENAI_COMPAT:=__disabled__} provider_type: remote::llama-openai-compat config: openai_compat_api_base: https://api.llama.com/compat/v1/ api_key: ${env.LLAMA_API_KEY} - - provider_id: ${env.ENABLE_TOGETHER_OPENAI_COMPAT:=__disabled__} - provider_type: remote::together-openai-compat - config: - openai_compat_api_base: https://api.together.xyz/v1 - api_key: ${env.TOGETHER_API_KEY} - - provider_id: ${env.ENABLE_GROQ_OPENAI_COMPAT:=__disabled__} - provider_type: remote::groq-openai-compat - config: - openai_compat_api_base: https://api.groq.com/openai/v1 - api_key: ${env.GROQ_API_KEY} - - provider_id: ${env.ENABLE_SAMBANOVA_OPENAI_COMPAT:=__disabled__} - provider_type: remote::sambanova-openai-compat - config: - openai_compat_api_base: https://api.sambanova.ai/v1 - api_key: ${env.SAMBANOVA_API_KEY} - - provider_id: ${env.ENABLE_CEREBRAS_OPENAI_COMPAT:=__disabled__} - provider_type: remote::cerebras-openai-compat - config: - openai_compat_api_base: https://api.cerebras.ai/v1 - api_key: ${env.CEREBRAS_API_KEY} - provider_id: ${env.ENABLE_SAMBANOVA:=__disabled__} provider_type: remote::sambanova config: