mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-31 04:43:52 +00:00
a193c9fc3f
This stubs in some OpenAI server-side compatibility with three new endpoints: /v1/openai/v1/models /v1/openai/v1/completions /v1/openai/v1/chat/completions This gives common inference apps using OpenAI clients the ability to talk to Llama Stack using an endpoint like http://localhost:8321/v1/openai/v1 . The two "v1" instances in there isn't awesome, but the thinking is that Llama Stack's API is v1 and then our OpenAI compatibility layer is compatible with OpenAI V1. And, some OpenAI clients implicitly assume the URL ends with "v1", so this gives maximum compatibility. The openai models endpoint is implemented in the routing layer, and just returns all the models Llama Stack knows about. The chat endpoints are only actually implemented for the remote-vllm provider right now, and it just proxies the completion and chat completion requests to the backend vLLM. The goal to support this for every inference provider - proxying directly to the provider's OpenAI endpoint for OpenAI-compatible providers. For providers that don't have an OpenAI-compatible API, we'll add a mixin to translate incoming OpenAI requests to Llama Stack inference requests and translate the Llama Stack inference responses to OpenAI responses.
93 lines
2.5 KiB
Python
93 lines
2.5 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
from enum import Enum
|
|
from typing import Any, Dict, List, Literal, Optional, Protocol, runtime_checkable
|
|
|
|
from openai.types.model import Model as OpenAIModel
|
|
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
from llama_stack.apis.resource import Resource, ResourceType
|
|
from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol
|
|
from llama_stack.schema_utils import json_schema_type, webmethod
|
|
|
|
|
|
class CommonModelFields(BaseModel):
|
|
metadata: Dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
description="Any additional metadata for this model",
|
|
)
|
|
|
|
|
|
@json_schema_type
|
|
class ModelType(str, Enum):
|
|
llm = "llm"
|
|
embedding = "embedding"
|
|
|
|
|
|
@json_schema_type
|
|
class Model(CommonModelFields, Resource):
|
|
type: Literal[ResourceType.model.value] = ResourceType.model.value
|
|
|
|
@property
|
|
def model_id(self) -> str:
|
|
return self.identifier
|
|
|
|
@property
|
|
def provider_model_id(self) -> str:
|
|
return self.provider_resource_id
|
|
|
|
model_config = ConfigDict(protected_namespaces=())
|
|
|
|
model_type: ModelType = Field(default=ModelType.llm)
|
|
|
|
|
|
class ModelInput(CommonModelFields):
|
|
model_id: str
|
|
provider_id: Optional[str] = None
|
|
provider_model_id: Optional[str] = None
|
|
model_type: Optional[ModelType] = ModelType.llm
|
|
model_config = ConfigDict(protected_namespaces=())
|
|
|
|
|
|
class ListModelsResponse(BaseModel):
|
|
data: List[Model]
|
|
|
|
|
|
class OpenAIListModelsResponse(BaseModel):
|
|
data: List[OpenAIModel]
|
|
|
|
|
|
@runtime_checkable
|
|
@trace_protocol
|
|
class Models(Protocol):
|
|
@webmethod(route="/models", method="GET")
|
|
async def list_models(self) -> ListModelsResponse: ...
|
|
|
|
@webmethod(route="/openai/v1/models", method="GET")
|
|
async def openai_list_models(self) -> OpenAIListModelsResponse: ...
|
|
|
|
@webmethod(route="/models/{model_id:path}", method="GET")
|
|
async def get_model(
|
|
self,
|
|
model_id: str,
|
|
) -> Model: ...
|
|
|
|
@webmethod(route="/models", method="POST")
|
|
async def register_model(
|
|
self,
|
|
model_id: str,
|
|
provider_model_id: Optional[str] = None,
|
|
provider_id: Optional[str] = None,
|
|
metadata: Optional[Dict[str, Any]] = None,
|
|
model_type: Optional[ModelType] = None,
|
|
) -> Model: ...
|
|
|
|
@webmethod(route="/models/{model_id:path}", method="DELETE")
|
|
async def unregister_model(
|
|
self,
|
|
model_id: str,
|
|
) -> None: ...
|