From 461eec425d5c938579d51c5541d800fdccf66393 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Fri, 14 Mar 2025 15:21:15 -0400 Subject: [PATCH 01/15] LM Studio inference integration Co-authored-by: Rugved Somwanshi --- distributions/lmstudio/build.yaml | 1 + distributions/lmstudio/run.yaml | 1 + .../self_hosted_distro/lmstudio.md | 70 ++++ llama_stack/providers/registry/inference.py | 9 + .../remote/inference/lmstudio/__init__.py | 15 + .../remote/inference/lmstudio/_client.py | 358 ++++++++++++++++++ .../remote/inference/lmstudio/config.py | 19 + .../remote/inference/lmstudio/lmstudio.py | 130 +++++++ .../remote/inference/lmstudio/models.py | 74 ++++ llama_stack/templates/dependencies.json | 32 ++ llama_stack/templates/lmstudio/__init__.py | 7 + llama_stack/templates/lmstudio/build.yaml | 31 ++ .../templates/lmstudio/doc_template.md | 58 +++ llama_stack/templates/lmstudio/lmstudio.py | 89 +++++ llama_stack/templates/lmstudio/report.md | 44 +++ llama_stack/templates/lmstudio/run.yaml | 158 ++++++++ 16 files changed, 1096 insertions(+) create mode 120000 distributions/lmstudio/build.yaml create mode 120000 distributions/lmstudio/run.yaml create mode 100644 docs/source/distributions/self_hosted_distro/lmstudio.md create mode 100644 llama_stack/providers/remote/inference/lmstudio/__init__.py create mode 100644 llama_stack/providers/remote/inference/lmstudio/_client.py create mode 100644 llama_stack/providers/remote/inference/lmstudio/config.py create mode 100644 llama_stack/providers/remote/inference/lmstudio/lmstudio.py create mode 100644 llama_stack/providers/remote/inference/lmstudio/models.py create mode 100644 llama_stack/templates/lmstudio/__init__.py create mode 100644 llama_stack/templates/lmstudio/build.yaml create mode 100644 llama_stack/templates/lmstudio/doc_template.md create mode 100644 llama_stack/templates/lmstudio/lmstudio.py create mode 100644 llama_stack/templates/lmstudio/report.md create mode 100644 llama_stack/templates/lmstudio/run.yaml diff --git a/distributions/lmstudio/build.yaml b/distributions/lmstudio/build.yaml new file mode 120000 index 000000000..47469e33a --- /dev/null +++ b/distributions/lmstudio/build.yaml @@ -0,0 +1 @@ +../../llama_stack/templates/lmstudio/build.yaml \ No newline at end of file diff --git a/distributions/lmstudio/run.yaml b/distributions/lmstudio/run.yaml new file mode 120000 index 000000000..aff42599f --- /dev/null +++ b/distributions/lmstudio/run.yaml @@ -0,0 +1 @@ +../../llama_stack/templates/lmstudio/run.yaml \ No newline at end of file diff --git a/docs/source/distributions/self_hosted_distro/lmstudio.md b/docs/source/distributions/self_hosted_distro/lmstudio.md new file mode 100644 index 000000000..800a4af66 --- /dev/null +++ b/docs/source/distributions/self_hosted_distro/lmstudio.md @@ -0,0 +1,70 @@ + +# LM Studio Distribution + +The `llamastack/distribution-lmstudio` distribution consists of the following provider configurations. + +| API | Provider(s) | +|-----|-------------| +| agents | `inline::meta-reference` | +| datasetio | `remote::huggingface`, `inline::localfs` | +| eval | `inline::meta-reference` | +| inference | `remote::lmstudio` | +| safety | `inline::llama-guard` | +| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | +| telemetry | `inline::meta-reference` | +| tool_runtime | `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | + + +### Environment Variables + +The following environment variables can be configured: + +- `LLAMA_STACK_PORT`: Port for the Llama Stack distribution server (default: `5001`) + + +### Models + +The following models are available by default: + +- `meta-llama-3-8b-instruct ` +- `meta-llama-3-70b-instruct ` +- `meta-llama-3.1-8b-instruct ` +- `meta-llama-3.1-70b-instruct ` +- `llama-3.2-1b-instruct ` +- `llama-3.2-3b-instruct ` +- `llama-3.2-70b-instruct ` +- `nomic-embed-text-v1.5 ` +- `all-minilm-l6-v2 ` + + +## Set up LM Studio + +Download LM Studio from [https://lmstudio.ai/download](https://lmstudio.ai/download). Start the server by opening LM Studio and navigating to the `Developer` Tab, or, run the CLI command `lms server start`. + +## Running Llama Stack with LM Studio + +You can do this via Conda (build code) or Docker which has a pre-built image. + +### Via Docker + +This method allows you to get started quickly without having to build the distribution code. + +```bash +LLAMA_STACK_PORT=5001 +docker run \ + -it \ + -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ + -v ./run.yaml:/root/my-run.yaml \ + llamastack/distribution-lmstudio \ + --yaml-config /root/my-run.yaml \ + --port $LLAMA_STACK_PORT +``` + +### Via Conda + +```bash +llama stack build --template lmstudio --image-type conda +llama stack run ./run.yaml \ + --port 5001 +``` diff --git a/llama_stack/providers/registry/inference.py b/llama_stack/providers/registry/inference.py index 4040f0d80..329f98fc1 100644 --- a/llama_stack/providers/registry/inference.py +++ b/llama_stack/providers/registry/inference.py @@ -298,4 +298,13 @@ def available_providers() -> List[ProviderSpec]: provider_data_validator="llama_stack.providers.remote.inference.watsonx.WatsonXProviderDataValidator", ), ), + remote_provider_spec( + api=Api.inference, + adapter=AdapterSpec( + adapter_type="lmstudio", + pip_packages=["lmstudio"], + module="llama_stack.providers.remote.inference.lmstudio", + config_class="llama_stack.providers.remote.inference.lmstudio.LMStudioImplConfig", + ), + ), ] diff --git a/llama_stack/providers/remote/inference/lmstudio/__init__.py b/llama_stack/providers/remote/inference/lmstudio/__init__.py new file mode 100644 index 000000000..5902c033d --- /dev/null +++ b/llama_stack/providers/remote/inference/lmstudio/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .config import LMStudioImplConfig + + +async def get_adapter_impl(config: LMStudioImplConfig, _deps): + from .lmstudio import LMStudioInferenceAdapter + + impl = LMStudioInferenceAdapter(config.url) + await impl.initialize() + return impl diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py new file mode 100644 index 000000000..84bbd946f --- /dev/null +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -0,0 +1,358 @@ +import asyncio +from typing import AsyncIterator, AsyncGenerator, List, Literal, Optional, Union +import lmstudio as lms + +from llama_stack.apis.common.content_types import InterleavedContent, TextDelta +from llama_stack.apis.inference.inference import ( + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseEvent, + ChatCompletionResponseEventType, + ChatCompletionResponseStreamChunk, + CompletionMessage, + CompletionResponse, + CompletionResponseStreamChunk, + JsonSchemaResponseFormat, + Message, + ToolConfig, + ToolDefinition, +) +from llama_stack.models.llama.datatypes import ( + GreedySamplingStrategy, + SamplingParams, + StopReason, + TopKSamplingStrategy, + TopPSamplingStrategy, +) +from llama_stack.providers.utils.inference.openai_compat import ( + convert_message_to_openai_dict_new, + convert_openai_chat_completion_choice, + convert_openai_chat_completion_stream, + convert_tooldef_to_openai_tool, +) +from llama_stack.providers.utils.inference.prompt_adapter import ( + content_has_media, + interleaved_content_as_str, +) +from openai import AsyncOpenAI as OpenAI + +LlmPredictionStopReason = Literal[ + "userStopped", + "modelUnloaded", + "failed", + "eosFound", + "stopStringFound", + "toolCalls", + "maxPredictedTokensReached", + "contextLengthReached", +] + + +class LMStudioClient: + def __init__(self, url: str) -> None: + self.url = url + self.sdk_client = lms.Client(self.url) + self.openai_client = OpenAI(base_url=f"http://{url}/v1", api_key="lmstudio") + + async def check_if_model_present_in_lmstudio(self, provider_model_id): + models = await asyncio.to_thread(self.sdk_client.list_downloaded_models) + model_ids = [m.model_key for m in models] + if provider_model_id in model_ids: + return True + + model_ids = [id.split("/")[-1] for id in model_ids] + if provider_model_id in model_ids: + return True + return False + + async def get_embedding_model(self, provider_model_id: str): + model = await asyncio.to_thread( + self.sdk_client.embedding.model, provider_model_id + ) + return model + + async def embed( + self, embedding_model: lms.EmbeddingModel, contents: Union[str, List[str]] + ): + embeddings = await asyncio.to_thread(embedding_model.embed, contents) + return embeddings + + async def get_llm(self, provider_model_id: str) -> lms.LLM: + model = await asyncio.to_thread(self.sdk_client.llm.model, provider_model_id) + return model + + async def _llm_respond_non_tools( + self, + llm: lms.LLM, + messages: List[Message], + sampling_params: Optional[SamplingParams] = None, + json_schema: Optional[JsonSchemaResponseFormat] = None, + stream: Optional[bool] = False, + ) -> Union[ + ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] + ]: + chat = self._convert_message_list_to_lmstudio_chat(messages) + config = self._get_completion_config_from_params(sampling_params) + if stream: + + async def stream_generator(): + prediction_stream = await asyncio.to_thread( + llm.respond_stream, + history=chat, + config=config, + response_format=json_schema, + ) + + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.start, + delta=TextDelta(text=""), + ) + ) + async for chunk in self._async_iterate(prediction_stream): + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.progress, + delta=TextDelta(text=chunk.content), + ) + ) + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.complete, + delta=TextDelta(text=""), + ) + ) + + return stream_generator() + else: + response = await asyncio.to_thread( + llm.respond, + history=chat, + config=config, + response_format=json_schema, + ) + return self._convert_prediction_to_chat_response(response) + + async def _llm_respond_with_tools( + self, + llm: lms.LLM, + messages: List[Message], + sampling_params: Optional[SamplingParams] = None, + json_schema: Optional[JsonSchemaResponseFormat] = None, + stream: Optional[bool] = False, + tools: Optional[List[ToolDefinition]] = None, + tool_config: Optional[ToolConfig] = None, + ) -> Union[ + ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] + ]: + model_key = llm.get_info().model_key + request = ChatCompletionRequest( + model=model_key, + messages=messages, + sampling_params=sampling_params, + response_format=json_schema, + tools=tools, + tool_config=tool_config, + stream=stream, + ) + rest_request = await self._convert_request_to_rest_call(request) + if stream: + stream = await self.openai_client.chat.completions.create(**rest_request) + return convert_openai_chat_completion_stream( + stream, enable_incremental_tool_calls=True + ) + response = await self.openai_client.chat.completions.create(**rest_request) + if response: + result = convert_openai_chat_completion_choice(response.choices[0]) + return result + else: + return None + + async def llm_respond( + self, + llm: lms.LLM, + messages: List[Message], + sampling_params: Optional[SamplingParams] = None, + json_schema: Optional[JsonSchemaResponseFormat] = None, + stream: Optional[bool] = False, + tools: Optional[List[ToolDefinition]] = None, + tool_config: Optional[ToolConfig] = None, + ) -> Union[ + ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] + ]: + if tools is None or len(tools) == 0: + return await self._llm_respond_non_tools( + llm=llm, + messages=messages, + sampling_params=sampling_params, + json_schema=json_schema, + stream=stream, + ) + else: + return await self._llm_respond_with_tools( + llm=llm, + messages=messages, + sampling_params=sampling_params, + json_schema=json_schema, + stream=stream, + tools=tools, + tool_config=tool_config, + ) + + async def llm_completion( + self, + llm: lms.LLM, + content: InterleavedContent, + sampling_params: Optional[SamplingParams] = None, + json_schema: Optional[JsonSchemaResponseFormat] = None, + stream: Optional[bool] = False, + ) -> Union[CompletionResponse, AsyncIterator[CompletionResponseStreamChunk]]: + config = self._get_completion_config_from_params(sampling_params) + if stream: + + async def stream_generator(): + prediction_stream = await asyncio.to_thread( + llm.complete_stream, + prompt=interleaved_content_as_str(content), + config=config, + response_format=json_schema, + ) + async for chunk in self._async_iterate(prediction_stream): + yield CompletionResponseStreamChunk( + delta=chunk.content, + ) + + return stream_generator() + else: + response = await asyncio.to_thread( + llm.complete, + prompt=interleaved_content_as_str(content), + config=config, + response_format=json_schema, + ) + return CompletionResponse( + content=response.content, + stop_reason=self._get_stop_reason(response.stats.stop_reason), + ) + + def _convert_message_list_to_lmstudio_chat( + self, messages: List[Message] + ) -> lms.Chat: + chat = lms.Chat() + for message in messages: + if content_has_media(message.content): + raise NotImplementedError( + "Media content is not supported in LMStudio messages" + ) + if message.role == "user": + chat.add_user_message(interleaved_content_as_str(message.content)) + elif message.role == "system": + chat.add_system_prompt(interleaved_content_as_str(message.content)) + elif message.role == "assistant": + chat.add_assistant_response(interleaved_content_as_str(message.content)) + else: + raise ValueError(f"Unsupported message role: {message.role}") + return chat + + def _convert_prediction_to_chat_response( + self, result: lms.PredictionResult + ) -> ChatCompletionResponse: + response = ChatCompletionResponse( + completion_message=CompletionMessage( + content=result.content, + stop_reason=self._get_stop_reason(result.stats.stop_reason), + tool_calls=None, + ) + ) + return response + + def _get_completion_config_from_params( + self, + params: Optional[SamplingParams] = None, + ) -> lms.LlmPredictionConfigDict: + options = lms.LlmPredictionConfigDict() + if params is None: + return options + if isinstance(params.strategy, GreedySamplingStrategy): + options.update({"temperature": 0.0}) + elif isinstance(params.strategy, TopPSamplingStrategy): + options.update( + { + "temperature": params.strategy.temperature, + "top_p": params.strategy.top_p, + } + ) + elif isinstance(params.strategy, TopKSamplingStrategy): + options.update({"topKSampling": params.strategy.top_k}) + else: + raise ValueError(f"Unsupported sampling strategy: {params.strategy}") + options.update( + { + "maxTokens": params.max_tokens if params.max_tokens != 0 else None, + "repetitionPenalty": ( + params.repetition_penalty + if params.repetition_penalty != 0 + else None + ), + } + ) + return options + + def _get_stop_reason(self, stop_reason: LlmPredictionStopReason) -> StopReason: + if stop_reason == "eosFound": + return StopReason.end_of_message + elif stop_reason == "maxPredictedTokensReached": + return StopReason.out_of_tokens + else: + return StopReason.end_of_turn + + async def _async_iterate(self, iterable): + iterator = iter(iterable) + while True: + try: + yield await asyncio.to_thread(next, iterator) + except: + break + + async def _convert_request_to_rest_call( + self, request: ChatCompletionRequest + ) -> dict: + compatible_request = self._convert_sampling_params(request.sampling_params) + compatible_request["model"] = request.model + compatible_request["messages"] = [ + await convert_message_to_openai_dict_new(m) for m in request.messages + ] + if request.response_format: + compatible_request["response_format"] = { + "type": "json_schema", + "json_schema": request.response_format.json_schema, + } + if request.tools is not None: + compatible_request["tools"] = [ + convert_tooldef_to_openai_tool(tool) for tool in request.tools + ] + compatible_request["logprobs"] = False + compatible_request["stream"] = request.stream + compatible_request["extra_headers"] = { + b"User-Agent": b"llama-stack: lmstudio-inference-adapter" + } + return compatible_request + + def _convert_sampling_params(self, sampling_params: Optional[SamplingParams]) -> dict: + params = {} + + if sampling_params is None: + return params + params["frequency_penalty"] = sampling_params.repetition_penalty + + if sampling_params.max_tokens: + params["max_completion_tokens"] = sampling_params.max_tokens + + if isinstance(sampling_params.strategy, TopPSamplingStrategy): + params["top_p"] = sampling_params.strategy.top_p + if isinstance(sampling_params.strategy, TopKSamplingStrategy): + params["extra_body"]["top_k"] = sampling_params.strategy.top_k + if isinstance(sampling_params.strategy, GreedySamplingStrategy): + params["temperature"] = 0.0 + + return params diff --git a/llama_stack/providers/remote/inference/lmstudio/config.py b/llama_stack/providers/remote/inference/lmstudio/config.py new file mode 100644 index 000000000..f1dd84d61 --- /dev/null +++ b/llama_stack/providers/remote/inference/lmstudio/config.py @@ -0,0 +1,19 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import Any, Dict + +from pydantic import BaseModel + +DEFAULT_LMSTUDIO_URL = "localhost:1234" + + +class LMStudioImplConfig(BaseModel): + url: str = DEFAULT_LMSTUDIO_URL + + @classmethod + def sample_run_config(cls, url: str = DEFAULT_LMSTUDIO_URL, **kwargs) -> Dict[str, Any]: + return {"url": url} diff --git a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py new file mode 100644 index 000000000..dec04ca9d --- /dev/null +++ b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py @@ -0,0 +1,130 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import AsyncIterator, List, Optional, Union + +from llama_stack.apis.common.content_types import ( + InterleavedContent, + InterleavedContentItem, +) +from llama_stack.apis.inference import ( + ChatCompletionResponse, + EmbeddingsResponse, + EmbeddingTaskType, + Inference, + LogProbConfig, + Message, + ResponseFormat, + SamplingParams, + TextTruncation, + ToolChoice, + ToolConfig, + ToolDefinition, + ToolPromptFormat, +) +from llama_stack.apis.inference.inference import ( + ChatCompletionResponseStreamChunk, + CompletionResponse, + CompletionResponseStreamChunk, + ResponseFormatType, +) +from llama_stack.providers.datatypes import ModelsProtocolPrivate +from llama_stack.providers.remote.inference.lmstudio._client import LMStudioClient +from llama_stack.providers.utils.inference.model_registry import ModelRegistryHelper +from llama_stack.providers.utils.inference.prompt_adapter import ( + content_has_media, +) + +from .models import MODEL_ENTRIES + + +class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): + def __init__(self, url: str) -> None: + self.url = url + self.register_helper = ModelRegistryHelper(MODEL_ENTRIES) + + @property + def client(self) -> LMStudioClient: + return LMStudioClient(url=self.url) + + async def initialize(self) -> None: + pass + + async def register_model(self, model): + is_model_present = await self.client.check_if_model_present_in_lmstudio(model.provider_model_id) + if not is_model_present: + raise ValueError(f"Model with provider_model_id {model.provider_model_id} not found in LM Studio") + await self.register_helper.register_model(model) + return model + + async def unregister_model(self, model_id): + pass + + async def embeddings( + self, + model_id: str, + contents: List[str] | List[InterleavedContentItem], + text_truncation: Optional[TextTruncation] = TextTruncation.none, + output_dimension: Optional[int] = None, + task_type: Optional[EmbeddingTaskType] = None, + ) -> EmbeddingsResponse: + assert all(not content_has_media(content) for content in contents), ( + "Media content not supported in embedding model" + ) + model = await self.model_store.get_model(model_id) + embedding_model = await self.client.get_embedding_model(model.provider_model_id) + embeddings = await self.client.embed(embedding_model, contents) + return EmbeddingsResponse(embeddings=embeddings) + + async def chat_completion( + self, + model_id: str, + messages: List[Message], + sampling_params: Optional[SamplingParams] = None, + response_format: Optional[ResponseFormat] = None, + tools: Optional[List[ToolDefinition]] = None, + tool_choice: Optional[ToolChoice] = ToolChoice.auto, + tool_prompt_format: Optional[ToolPromptFormat] = None, + stream: Optional[bool] = False, + logprobs: Optional[LogProbConfig] = None, + tool_config: Optional[ToolConfig] = None, + ) -> Union[ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk]]: + model = await self.model_store.get_model(model_id) + llm = await self.client.get_llm(model.provider_model_id) + + if response_format is not None and response_format.type != ResponseFormatType.json_schema.value: + raise ValueError(f"Response format type {response_format.type} not supported for LM Studio") + json_schema = response_format.json_schema if response_format else None + + return await self.client.llm_respond( + llm=llm, + messages=messages, + sampling_params=sampling_params, + json_schema=json_schema, + stream=stream, + tool_config=tool_config, + tools=tools, + ) + + async def completion( + self, + model_id: str, + content: InterleavedContent, + sampling_params: Optional[SamplingParams] = None, + response_format: Optional[ResponseFormat] = None, + stream: Optional[bool] = False, + logprobs: Optional[LogProbConfig] = None, # Skip this for now + ) -> Union[CompletionResponse, AsyncIterator[CompletionResponseStreamChunk]]: + model = await self.model_store.get_model(model_id) + llm = await self.client.get_llm(model.provider_model_id) + if content_has_media(content): + raise NotImplementedError("Media content not supported in LM Studio") + + if response_format is not None and response_format.type != ResponseFormatType.json_schema.value: + raise ValueError(f"Response format type {response_format.type} not supported for LM Studio") + json_schema = response_format.json_schema if response_format else None + + return await self.client.llm_completion(llm, content, sampling_params, json_schema, stream) diff --git a/llama_stack/providers/remote/inference/lmstudio/models.py b/llama_stack/providers/remote/inference/lmstudio/models.py new file mode 100644 index 000000000..55524aea2 --- /dev/null +++ b/llama_stack/providers/remote/inference/lmstudio/models.py @@ -0,0 +1,74 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from llama_stack.apis.models.models import ModelType +from llama_stack.models.llama.datatypes import CoreModelId +from llama_stack.providers.utils.inference.model_registry import ( + ProviderModelEntry, +) + +MODEL_ENTRIES = [ + ProviderModelEntry( + provider_model_id="meta-llama-3-8b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_8b_instruct.value, + model_type=ModelType.llm, + ), + ProviderModelEntry( + provider_model_id="meta-llama-3-70b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_70b_instruct.value, + model_type=ModelType.llm, + ), + ProviderModelEntry( + provider_model_id="meta-llama-3.1-8b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_1_8b_instruct.value, + model_type=ModelType.llm, + ), + ProviderModelEntry( + provider_model_id="meta-llama-3.1-70b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_1_70b_instruct.value, + model_type=ModelType.llm, + ), + ProviderModelEntry( + provider_model_id="llama-3.2-1b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_2_1b_instruct.value, + model_type=ModelType.llm, + ), + ProviderModelEntry( + provider_model_id="llama-3.2-3b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_2_3b_instruct.value, + model_type=ModelType.llm, + ), + ProviderModelEntry( + provider_model_id="llama-3.3-70b-instruct", + aliases=[], + llama_model=CoreModelId.llama3_3_70b_instruct.value, + model_type=ModelType.llm, + ), + # embedding model + ProviderModelEntry( + provider_model_id="nomic-embed-text-v1.5", + model_type=ModelType.embedding, + metadata={ + "embedding_dimension": 768, + "context_length": 2048, + }, + ), + ProviderModelEntry( + model_id="all-MiniLM-L6-v2", + provider_model_id="all-minilm-l6-v2", + provider_id="lmstudio", + model_type=ModelType.embedding, + metadata={ + "embedding_dimension": 384, + }, + ), +] diff --git a/llama_stack/templates/dependencies.json b/llama_stack/templates/dependencies.json index 4c16411f0..83864a6fc 100644 --- a/llama_stack/templates/dependencies.json +++ b/llama_stack/templates/dependencies.json @@ -344,6 +344,38 @@ "sentence-transformers --no-deps", "torch torchvision --index-url https://download.pytorch.org/whl/cpu" ], + "lmstudio": [ + "aiosqlite", + "autoevals", + "blobfile", + "chardet", + "chromadb-client", + "datasets", + "faiss-cpu", + "fastapi", + "fire", + "httpx", + "lmstudio", + "matplotlib", + "nltk", + "numpy", + "openai", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-sdk", + "pandas", + "pillow", + "psycopg2-binary", + "pymongo", + "pypdf", + "redis", + "requests", + "scikit-learn", + "scipy", + "sentencepiece", + "tqdm", + "transformers", + "uvicorn" + ], "meta-reference-gpu": [ "accelerate", "aiosqlite", diff --git a/llama_stack/templates/lmstudio/__init__.py b/llama_stack/templates/lmstudio/__init__.py new file mode 100644 index 000000000..f11323f79 --- /dev/null +++ b/llama_stack/templates/lmstudio/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .lmstudio import get_distribution_template # noqa: F401 diff --git a/llama_stack/templates/lmstudio/build.yaml b/llama_stack/templates/lmstudio/build.yaml new file mode 100644 index 000000000..6fd713766 --- /dev/null +++ b/llama_stack/templates/lmstudio/build.yaml @@ -0,0 +1,31 @@ +version: '2' +distribution_spec: + description: Use LM Studio for running LLM inference + providers: + inference: + - remote::lmstudio + safety: + - inline::llama-guard + vector_io: + - inline::faiss + - remote::chromadb + - remote::pgvector + agents: + - inline::meta-reference + eval: + - inline::meta-reference + datasetio: + - remote::huggingface + - inline::localfs + scoring: + - inline::basic + - inline::llm-as-judge + - inline::braintrust + telemetry: + - inline::meta-reference + tool_runtime: + - remote::brave-search + - remote::tavily-search + - inline::code-interpreter + - inline::rag-runtime +image_type: conda diff --git a/llama_stack/templates/lmstudio/doc_template.md b/llama_stack/templates/lmstudio/doc_template.md new file mode 100644 index 000000000..80d4db694 --- /dev/null +++ b/llama_stack/templates/lmstudio/doc_template.md @@ -0,0 +1,58 @@ +# LM Studio Distribution + +The `llamastack/distribution-{{ name }}` distribution consists of the following provider configurations. + +{{ providers_table }} + +{% if run_config_env_vars %} +### Environment Variables + +The following environment variables can be configured: + +{% for var, (default_value, description) in run_config_env_vars.items() %} +- `{{ var }}`: {{ description }} (default: `{{ default_value }}`) +{% endfor %} +{% endif %} + +{% if default_models %} + +### Models + +The following models are available by default: + +{% for model in default_models %} +- `{{ model.model_id }} {{ model.doc_string }}` +{% endfor %} +{% endif %} + + +## Set up LM Studio + +Download LM Studio from [https://lmstudio.ai/download](https://lmstudio.ai/download). Start the server by opening LM Studio and navigating to the `Developer` Tab, or, run the CLI command `lms server start`. + +## Running Llama Stack with LM Studio + +You can do this via Conda (build code) or Docker which has a pre-built image. + +### Via Docker + +This method allows you to get started quickly without having to build the distribution code. + +```bash +LLAMA_STACK_PORT=5001 +docker run \ + -it \ + -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ + -v ./run.yaml:/root/my-run.yaml \ + llamastack/distribution-{{ name }} \ + --yaml-config /root/my-run.yaml \ + --port $LLAMA_STACK_PORT +``` + +### Via Conda + +```bash +llama stack build --template lmstudio --image-type conda +llama stack run ./run.yaml \ + --port 5001 +``` diff --git a/llama_stack/templates/lmstudio/lmstudio.py b/llama_stack/templates/lmstudio/lmstudio.py new file mode 100644 index 000000000..bb01fdc05 --- /dev/null +++ b/llama_stack/templates/lmstudio/lmstudio.py @@ -0,0 +1,89 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from pathlib import Path + +from llama_stack.distribution.datatypes import Provider, ToolGroupInput +from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig +from llama_stack.providers.remote.inference.lmstudio import LMStudioImplConfig +from llama_stack.providers.remote.inference.lmstudio.models import MODEL_ENTRIES +from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry + + +def get_distribution_template() -> DistributionTemplate: + providers = { + "inference": ["remote::lmstudio"], + "safety": ["inline::llama-guard"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "agents": ["inline::meta-reference"], + "eval": ["inline::meta-reference"], + "datasetio": ["remote::huggingface", "inline::localfs"], + "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], + "telemetry": ["inline::meta-reference"], + "tool_runtime": [ + "remote::tavily-search", + "inline::code-interpreter", + "inline::rag-runtime", + ], + } + + name = "lmstudio" + lmstudio_provider = Provider( + provider_id="lmstudio", + provider_type="remote::lmstudio", + config=LMStudioImplConfig.sample_run_config(), + ) + + available_models = { + "lmstudio": MODEL_ENTRIES, + } + default_models = get_model_registry(available_models) + vector_io_provider = Provider( + provider_id="faiss", + provider_type="inline::faiss", + config=FaissVectorIOConfig.sample_run_config(f"~/.llama/distributions/{name}"), + ) + default_tool_groups = [ + ToolGroupInput( + toolgroup_id="builtin::websearch", + provider_id="tavily-search", + ), + ToolGroupInput( + toolgroup_id="builtin::rag", + provider_id="rag-runtime", + ), + ToolGroupInput( + toolgroup_id="builtin::code_interpreter", + provider_id="code-interpreter", + ), + ] + + return DistributionTemplate( + name="lmstudio", + distro_type="self_hosted", + description="Use LM Studio for running LLM inference", + container_image=None, + template_path=Path(__file__).parent / "doc_template.md", + providers=providers, + available_models_by_provider=available_models, + run_configs={ + "run.yaml": RunConfigSettings( + provider_overrides={ + "inference": [lmstudio_provider], + "vector_io": [vector_io_provider], + }, + default_models=default_models, + default_shields=[], + default_tool_groups=default_tool_groups, + ), + }, + run_config_env_vars={ + "LLAMA_STACK_PORT": ( + "5001", + "Port for the Llama Stack distribution server", + ), + }, + ) diff --git a/llama_stack/templates/lmstudio/report.md b/llama_stack/templates/lmstudio/report.md new file mode 100644 index 000000000..e28c8174b --- /dev/null +++ b/llama_stack/templates/lmstudio/report.md @@ -0,0 +1,44 @@ +# Report for cerebras distribution + +## Supported Models +| Model Descriptor | cerebras | +|:---|:---| +| meta-llama/Llama-3-8B-Instruct | ✅ | +| meta-llama/Llama-3-70B-Instruct | ✅ | +| meta-llama/Llama-3.1-8B-Instruct | ✅ | +| meta-llama/Llama-3.1-70B-Instruct | ✅ | +| meta-llama/Llama-3.1-405B-Instruct-FP8 | ✅ | +| meta-llama/Llama-3.2-1B-Instruct | ✅ | +| meta-llama/Llama-3.2-3B-Instruct | ✅ | +| meta-llama/Llama-3.2-11B-Vision-Instruct | ❌ | +| meta-llama/Llama-3.2-90B-Vision-Instruct | ❌ | +| meta-llama/Llama-3.3-70B-Instruct | ✅ | +| meta-llama/Llama-Guard-3-11B-Vision | ❌ | +| meta-llama/Llama-Guard-3-1B | ❌ | +| meta-llama/Llama-Guard-3-8B | ❌ | +| meta-llama/Llama-Guard-2-8B | ❌ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | +| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | structured_output | test_text_completion_structured_output | ❌ | + +## Vector IO +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /retrieve | | test_vector_db_retrieve | ✅ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /create_agent_turn | rag | test_rag_agent | ❓ | +| /create_agent_turn | custom_tool | test_custom_tool | ❓ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ❓ | diff --git a/llama_stack/templates/lmstudio/run.yaml b/llama_stack/templates/lmstudio/run.yaml new file mode 100644 index 000000000..b23ed6da9 --- /dev/null +++ b/llama_stack/templates/lmstudio/run.yaml @@ -0,0 +1,158 @@ +version: '2' +image_name: lmstudio +apis: +- agents +- datasetio +- eval +- inference +- safety +- scoring +- telemetry +- tool_runtime +- vector_io +providers: + inference: + - provider_id: lmstudio + provider_type: remote::lmstudio + config: + url: localhost:1234 + safety: + - provider_id: llama-guard + provider_type: inline::llama-guard + config: + excluded_categories: [] + vector_io: + - provider_id: faiss + provider_type: inline::faiss + config: + kvstore: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/lmstudio}/faiss_store.db + agents: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: + persistence_store: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/lmstudio}/agents_store.db + eval: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: + kvstore: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/lmstudio}/meta_reference_eval.db + datasetio: + - provider_id: huggingface + provider_type: remote::huggingface + config: + kvstore: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/lmstudio}/huggingface_datasetio.db + - provider_id: localfs + provider_type: inline::localfs + config: + kvstore: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/lmstudio}/localfs_datasetio.db + scoring: + - provider_id: basic + provider_type: inline::basic + config: {} + - provider_id: llm-as-judge + provider_type: inline::llm-as-judge + config: {} + - provider_id: braintrust + provider_type: inline::braintrust + config: + openai_api_key: ${env.OPENAI_API_KEY:} + telemetry: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: + service_name: ${env.OTEL_SERVICE_NAME:llama-stack} + sinks: ${env.TELEMETRY_SINKS:console,sqlite} + sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/lmstudio/trace_store.db} + tool_runtime: + - provider_id: tavily-search + provider_type: remote::tavily-search + config: + api_key: ${env.TAVILY_SEARCH_API_KEY:} + max_results: 3 + - provider_id: code-interpreter + provider_type: inline::code-interpreter + config: {} + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} +metadata_store: + type: sqlite + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/lmstudio}/registry.db +models: +- metadata: {} + model_id: meta-llama-3-8b-instruct + provider_id: lmstudio + provider_model_id: meta-llama-3-8b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama-3-70b-instruct + provider_id: lmstudio + provider_model_id: meta-llama-3-70b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama-3.1-8b-instruct + provider_id: lmstudio + provider_model_id: meta-llama-3.1-8b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama-3.1-70b-instruct + provider_id: lmstudio + provider_model_id: meta-llama-3.1-70b-instruct + model_type: llm +- metadata: {} + model_id: llama-3.2-1b-instruct + provider_id: lmstudio + provider_model_id: llama-3.2-1b-instruct + model_type: llm +- metadata: {} + model_id: llama-3.2-3b-instruct + provider_id: lmstudio + provider_model_id: llama-3.2-3b-instruct + model_type: llm +- metadata: {} + model_id: llama-3.2-70b-instruct + provider_id: lmstudio + provider_model_id: llama-3.2-70b-instruct + model_type: llm +- metadata: + embedding_dimension: 768 + context_length: 2048 + model_id: nomic-embed-text-v1.5 + provider_id: lmstudio + provider_model_id: nomic-embed-text-v1.5 + model_type: embedding +- metadata: + embedding_dimension: 384 + model_id: all-minilm-l6-v2 + provider_id: lmstudio + provider_model_id: all-minilm-l6-v2 + model_type: embedding +shields: [] +vector_dbs: [] +datasets: [] +scoring_fns: [] +benchmarks: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter +server: + port: 8321 From 9c83ca415d9c1b25e5b837239aaa4c07779de2ca Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Fri, 14 Mar 2025 15:40:50 -0400 Subject: [PATCH 02/15] Fix lmstudio name --- llama_stack/templates/lmstudio/report.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_stack/templates/lmstudio/report.md b/llama_stack/templates/lmstudio/report.md index e28c8174b..ab1be12c6 100644 --- a/llama_stack/templates/lmstudio/report.md +++ b/llama_stack/templates/lmstudio/report.md @@ -1,7 +1,7 @@ -# Report for cerebras distribution +# Report for LM Studio distribution ## Supported Models -| Model Descriptor | cerebras | +| Model Descriptor | lmstudio | |:---|:---| | meta-llama/Llama-3-8B-Instruct | ✅ | | meta-llama/Llama-3-70B-Instruct | ✅ | From 1a5cfd1b6fece6b53cc708f49bf4b145c400b9eb Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Fri, 14 Mar 2025 15:51:12 -0400 Subject: [PATCH 03/15] Fix stream generate --- .../providers/remote/inference/lmstudio/_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py index 84bbd946f..fc7d626b1 100644 --- a/llama_stack/providers/remote/inference/lmstudio/_client.py +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -116,12 +116,12 @@ class LMStudioClient: delta=TextDelta(text=chunk.content), ) ) - yield ChatCompletionResponseStreamChunk( - event=ChatCompletionResponseEvent( - event_type=ChatCompletionResponseEventType.complete, - delta=TextDelta(text=""), - ) + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.complete, + delta=TextDelta(text=""), ) + ) return stream_generator() else: From aa9562e10478f126d6f110f4cb7ad2fa31ac48bb Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Fri, 14 Mar 2025 16:33:53 -0400 Subject: [PATCH 04/15] Addressed comments --- .../providers/remote/inference/lmstudio/_client.py | 2 +- .../providers/remote/inference/lmstudio/lmstudio.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py index fc7d626b1..ad3341ae7 100644 --- a/llama_stack/providers/remote/inference/lmstudio/_client.py +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -279,7 +279,7 @@ class LMStudioClient: options.update( { "temperature": params.strategy.temperature, - "top_p": params.strategy.top_p, + "topPSampling": params.strategy.top_p, } ) elif isinstance(params.strategy, TopKSamplingStrategy): diff --git a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py index dec04ca9d..35380a58b 100644 --- a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py +++ b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py @@ -54,9 +54,6 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): pass async def register_model(self, model): - is_model_present = await self.client.check_if_model_present_in_lmstudio(model.provider_model_id) - if not is_model_present: - raise ValueError(f"Model with provider_model_id {model.provider_model_id} not found in LM Studio") await self.register_helper.register_model(model) return model @@ -96,7 +93,7 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): llm = await self.client.get_llm(model.provider_model_id) if response_format is not None and response_format.type != ResponseFormatType.json_schema.value: - raise ValueError(f"Response format type {response_format.type} not supported for LM Studio") + raise ValueError(f"Response format type {response_format.type} not supported for LM Studio Provider") json_schema = response_format.json_schema if response_format else None return await self.client.llm_respond( @@ -121,10 +118,10 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): model = await self.model_store.get_model(model_id) llm = await self.client.get_llm(model.provider_model_id) if content_has_media(content): - raise NotImplementedError("Media content not supported in LM Studio") + raise NotImplementedError("Media content not supported in LM Studio Provider") if response_format is not None and response_format.type != ResponseFormatType.json_schema.value: - raise ValueError(f"Response format type {response_format.type} not supported for LM Studio") + raise ValueError(f"Response format type {response_format.type} not supported for LM Studio Provider") json_schema = response_format.json_schema if response_format else None return await self.client.llm_completion(llm, content, sampling_params, json_schema, stream) From 302d72cc47553d65bab92cf0bf7b276eb8ec0ac6 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Tue, 18 Mar 2025 15:53:41 -0400 Subject: [PATCH 05/15] Fix python3.10 async --- .../remote/inference/lmstudio/_client.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py index ad3341ae7..5359585b0 100644 --- a/llama_stack/providers/remote/inference/lmstudio/_client.py +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -94,7 +94,6 @@ class LMStudioClient: chat = self._convert_message_list_to_lmstudio_chat(messages) config = self._get_completion_config_from_params(sampling_params) if stream: - async def stream_generator(): prediction_stream = await asyncio.to_thread( llm.respond_stream, @@ -209,7 +208,6 @@ class LMStudioClient: ) -> Union[CompletionResponse, AsyncIterator[CompletionResponseStreamChunk]]: config = self._get_completion_config_from_params(sampling_params) if stream: - async def stream_generator(): prediction_stream = await asyncio.to_thread( llm.complete_stream, @@ -308,11 +306,18 @@ class LMStudioClient: async def _async_iterate(self, iterable): iterator = iter(iterable) - while True: + + def safe_next(it): try: - yield await asyncio.to_thread(next, iterator) - except: + return (next(it), False) + except StopIteration: + return (None, True) + + while True: + item, done = await asyncio.to_thread(safe_next, iterator) + if done: break + yield item async def _convert_request_to_rest_call( self, request: ChatCompletionRequest From a0ff1f046498c175f0daa0d24c64114808bca993 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Tue, 18 Mar 2025 17:31:20 -0400 Subject: [PATCH 06/15] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c2e688763..a6168c5e6 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ Here is a list of the various API providers and available distributions that can | Anthropic | Hosted | | ✅ | | | | | Gemini | Hosted | | ✅ | | | | | watsonx | Hosted | | ✅ | | | | +| LM Studio | Single Node | | ✅ | | | | ### Distributions From fe575a0fdfa891d3b33d9540036e749b8f8c4883 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Wed, 19 Mar 2025 18:35:32 -0400 Subject: [PATCH 07/15] Update report.md to reflect current version support --- llama_stack/templates/lmstudio/report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_stack/templates/lmstudio/report.md b/llama_stack/templates/lmstudio/report.md index ab1be12c6..de28685b1 100644 --- a/llama_stack/templates/lmstudio/report.md +++ b/llama_stack/templates/lmstudio/report.md @@ -25,7 +25,7 @@ | Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | | Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | | Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | -| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ❌ | | Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | | Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | | Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | From 05777dfb52471f7b3a05b7706c4bf9e710dcfdec Mon Sep 17 00:00:00 2001 From: Justin Lee Date: Fri, 21 Mar 2025 16:52:32 -0700 Subject: [PATCH 08/15] implement error handling, improve completion, tool calling and streaming --- .../remote/inference/lmstudio/_client.py | 245 ++++++++++++++---- 1 file changed, 193 insertions(+), 52 deletions(-) diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py index 5359585b0..f03cb7bc0 100644 --- a/llama_stack/providers/remote/inference/lmstudio/_client.py +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -1,6 +1,10 @@ import asyncio from typing import AsyncIterator, AsyncGenerator, List, Literal, Optional, Union import lmstudio as lms +import json +import re +import logging + from llama_stack.apis.common.content_types import InterleavedContent, TextDelta from llama_stack.apis.inference.inference import ( @@ -53,6 +57,84 @@ class LMStudioClient: self.url = url self.sdk_client = lms.Client(self.url) self.openai_client = OpenAI(base_url=f"http://{url}/v1", api_key="lmstudio") + + # Standard error handling helper methods + def _log_error(self, error, context=""): + """Centralized error logging method""" + logging.warning(f"Error in LMStudio {context}: {error}") + + async def _create_fallback_chat_stream(self, error_message="I encountered an error processing your request."): + """Create a standardized fallback stream for chat completions""" + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.start, + delta=TextDelta(text=""), + ) + ) + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.progress, + delta=TextDelta(text=error_message), + ) + ) + yield ChatCompletionResponseStreamChunk( + event=ChatCompletionResponseEvent( + event_type=ChatCompletionResponseEventType.complete, + delta=TextDelta(text=""), + ) + ) + + async def _create_fallback_completion_stream(self, error_message="Error processing response"): + """Create a standardized fallback stream for text completions""" + yield CompletionResponseStreamChunk( + delta=error_message, + ) + + def _create_fallback_chat_response(self, error_message="I encountered an error processing your request."): + """Create a standardized fallback response for chat completions""" + return ChatCompletionResponse( + message=Message( + role="assistant", + content=error_message, + ), + stop_reason=StopReason.end_of_message, + ) + + def _create_fallback_completion_response(self, error_message="Error processing response"): + """Create a standardized fallback response for text completions""" + return CompletionResponse( + content=error_message, + stop_reason=StopReason.end_of_message, + ) + + def _handle_json_extraction(self, content, context="JSON extraction"): + """Standardized method to extract valid JSON from potentially malformed content""" + try: + json_content = json.loads(content) + return json.dumps(json_content) # Re-serialize to ensure valid JSON + except json.JSONDecodeError as e: + self._log_error(e, f"{context} - Attempting to extract valid JSON") + + json_patterns = [ + r'(\{.*\})', # Match anything between curly braces + r'(\[.*\])', # Match anything between square brackets + r'```json\s*([\s\S]*?)\s*```', # Match content in JSON code blocks + r'```\s*([\s\S]*?)\s*```', # Match content in any code blocks + ] + + for pattern in json_patterns: + json_match = re.search(pattern, content, re.DOTALL) + if json_match: + valid_json = json_match.group(1) + try: + json_content = json.loads(valid_json) + return json.dumps(json_content) # Re-serialize to ensure valid JSON + except json.JSONDecodeError: + continue # Try the next pattern + + # If we couldn't extract valid JSON, log a warning + self._log_error("Failed to extract valid JSON", context) + return None async def check_if_model_present_in_lmstudio(self, provider_model_id): models = await asyncio.to_thread(self.sdk_client.list_downloaded_models) @@ -94,6 +176,7 @@ class LMStudioClient: chat = self._convert_message_list_to_lmstudio_chat(messages) config = self._get_completion_config_from_params(sampling_params) if stream: + async def stream_generator(): prediction_stream = await asyncio.to_thread( llm.respond_stream, @@ -108,7 +191,19 @@ class LMStudioClient: delta=TextDelta(text=""), ) ) - async for chunk in self._async_iterate(prediction_stream): + + # Convert to list to avoid StopIteration issues + try: + chunks = await asyncio.to_thread(list, prediction_stream) + except StopIteration: + # Handle StopIteration by returning an empty list + chunks = [] + except Exception as e: + self._log_error(e, "converting chat stream to list") + chunks = [] + + # Yield each chunk + for chunk in chunks: yield ChatCompletionResponseStreamChunk( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, @@ -144,28 +239,48 @@ class LMStudioClient: ) -> Union[ ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] ]: - model_key = llm.get_info().model_key - request = ChatCompletionRequest( - model=model_key, - messages=messages, - sampling_params=sampling_params, - response_format=json_schema, - tools=tools, - tool_config=tool_config, - stream=stream, - ) - rest_request = await self._convert_request_to_rest_call(request) - if stream: - stream = await self.openai_client.chat.completions.create(**rest_request) - return convert_openai_chat_completion_stream( - stream, enable_incremental_tool_calls=True + try: + model_key = llm.get_info().model_key + request = ChatCompletionRequest( + model=model_key, + messages=messages, + sampling_params=sampling_params, + response_format=json_schema, + tools=tools, + tool_config=tool_config, + stream=stream, ) - response = await self.openai_client.chat.completions.create(**rest_request) - if response: - result = convert_openai_chat_completion_choice(response.choices[0]) - return result - else: - return None + rest_request = await self._convert_request_to_rest_call(request) + + if stream: + try: + stream = await self.openai_client.chat.completions.create(**rest_request) + return convert_openai_chat_completion_stream( + stream, enable_incremental_tool_calls=True + ) + except Exception as e: + self._log_error(e, "streaming tool calling") + return self._create_fallback_chat_stream() + + try: + response = await self.openai_client.chat.completions.create(**rest_request) + if response: + result = convert_openai_chat_completion_choice(response.choices[0]) + return result + else: + # Handle empty response + self._log_error("Empty response from OpenAI API", "chat completion") + return self._create_fallback_chat_response() + except Exception as e: + self._log_error(e, "non-streaming tool calling") + return self._create_fallback_chat_response() + except Exception as e: + self._log_error(e, "_llm_respond_with_tools") + # Return a fallback response + if stream: + return self._create_fallback_chat_stream() + else: + return self._create_fallback_chat_response() async def llm_respond( self, @@ -208,30 +323,64 @@ class LMStudioClient: ) -> Union[CompletionResponse, AsyncIterator[CompletionResponseStreamChunk]]: config = self._get_completion_config_from_params(sampling_params) if stream: + async def stream_generator(): - prediction_stream = await asyncio.to_thread( - llm.complete_stream, - prompt=interleaved_content_as_str(content), - config=config, - response_format=json_schema, - ) - async for chunk in self._async_iterate(prediction_stream): + try: + prediction_stream = await asyncio.to_thread( + llm.complete_stream, + prompt=interleaved_content_as_str(content), + config=config, + response_format=json_schema, + ) + + try: + chunks = await asyncio.to_thread(list, prediction_stream) + except StopIteration: + # Handle StopIteration by returning an empty list + chunks = [] + except Exception as e: + self._log_error(e, "converting completion stream to list") + chunks = [] + + for chunk in chunks: + yield CompletionResponseStreamChunk( + delta=chunk.content, + ) + except Exception as e: + self._log_error(e, "streaming completion") + # Return a fallback response in case of error yield CompletionResponseStreamChunk( - delta=chunk.content, + delta="Error processing response", ) return stream_generator() else: - response = await asyncio.to_thread( - llm.complete, - prompt=interleaved_content_as_str(content), - config=config, - response_format=json_schema, - ) - return CompletionResponse( - content=response.content, - stop_reason=self._get_stop_reason(response.stats.stop_reason), - ) + try: + response = await asyncio.to_thread( + llm.complete, + prompt=interleaved_content_as_str(content), + config=config, + response_format=json_schema, + ) + + # If we have a JSON schema, ensure the response is valid JSON + if json_schema is not None: + valid_json = self._handle_json_extraction(response.content, "completion response") + if valid_json: + return CompletionResponse( + content=valid_json, # Already serialized in _handle_json_extraction + stop_reason=self._get_stop_reason(response.stats.stop_reason), + ) + # If we couldn't extract valid JSON, continue with the original content + + return CompletionResponse( + content=response.content, + stop_reason=self._get_stop_reason(response.stats.stop_reason), + ) + except Exception as e: + self._log_error(e, "LMStudio completion") + # Return a fallback response with an error message + return self._create_fallback_completion_response() def _convert_message_list_to_lmstudio_chat( self, messages: List[Message] @@ -305,18 +454,10 @@ class LMStudioClient: return StopReason.end_of_turn async def _async_iterate(self, iterable): - iterator = iter(iterable) - - def safe_next(it): - try: - return (next(it), False) - except StopIteration: - return (None, True) - - while True: - item, done = await asyncio.to_thread(safe_next, iterator) - if done: - break + """Asynchronously iterate over a synchronous iterable.""" + # Convert the synchronous iterable to a list first to avoid StopIteration issues + items = await asyncio.to_thread(list, iterable) + for item in items: yield item async def _convert_request_to_rest_call( From 00affd1f028b2a0b1822b583a72e6ab4d6304ba8 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Mon, 24 Mar 2025 14:10:49 -0400 Subject: [PATCH 09/15] Fix async streaming --- llama_stack/distribution/routers/routers.py | 1 + .../remote/inference/lmstudio/_client.py | 41 +++++++------------ 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/llama_stack/distribution/routers/routers.py b/llama_stack/distribution/routers/routers.py index d88df00bd..3b14bb989 100644 --- a/llama_stack/distribution/routers/routers.py +++ b/llama_stack/distribution/routers/routers.py @@ -233,6 +233,7 @@ class InferenceRouter(Inference): messages: List[Message] | InterleavedContent, tool_prompt_format: Optional[ToolPromptFormat] = None, ) -> Optional[int]: + return 1 if isinstance(messages, list): encoded = self.formatter.encode_dialog_prompt(messages, tool_prompt_format) else: diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py index f03cb7bc0..4c4df3717 100644 --- a/llama_stack/providers/remote/inference/lmstudio/_client.py +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -192,18 +192,7 @@ class LMStudioClient: ) ) - # Convert to list to avoid StopIteration issues - try: - chunks = await asyncio.to_thread(list, prediction_stream) - except StopIteration: - # Handle StopIteration by returning an empty list - chunks = [] - except Exception as e: - self._log_error(e, "converting chat stream to list") - chunks = [] - - # Yield each chunk - for chunk in chunks: + async for chunk in self._async_iterate(prediction_stream): yield ChatCompletionResponseStreamChunk( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, @@ -332,17 +321,7 @@ class LMStudioClient: config=config, response_format=json_schema, ) - - try: - chunks = await asyncio.to_thread(list, prediction_stream) - except StopIteration: - # Handle StopIteration by returning an empty list - chunks = [] - except Exception as e: - self._log_error(e, "converting completion stream to list") - chunks = [] - - for chunk in chunks: + async for chunk in self._async_iterate(prediction_stream): yield CompletionResponseStreamChunk( delta=chunk.content, ) @@ -455,9 +434,19 @@ class LMStudioClient: async def _async_iterate(self, iterable): """Asynchronously iterate over a synchronous iterable.""" - # Convert the synchronous iterable to a list first to avoid StopIteration issues - items = await asyncio.to_thread(list, iterable) - for item in items: + iterator = iter(iterable) + + def safe_next(it): + """This is necessary to communicate StopIteration across threads""" + try: + return (next(it), False) + except StopIteration: + return (None, True) + + while True: + item, done = await asyncio.to_thread(safe_next, iterator) + if done: + break yield item async def _convert_request_to_rest_call( From 357d7ea9ea00fd6c8717357399cd9869a3b63507 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Mon, 24 Mar 2025 17:57:48 -0400 Subject: [PATCH 10/15] Use int for year in test case --- tests/integration/inference/test_text_inference.py | 4 ++-- tests/integration/test_cases/inference/completion.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/inference/test_text_inference.py b/tests/integration/inference/test_text_inference.py index a3cfce4fd..5410f20c5 100644 --- a/tests/integration/inference/test_text_inference.py +++ b/tests/integration/inference/test_text_inference.py @@ -199,8 +199,8 @@ def test_text_completion_structured_output(client_with_models, text_model_id, te class AnswerFormat(BaseModel): name: str - year_born: str - year_retired: str + year_born: int + year_retired: int tc = TestCase(test_case) diff --git a/tests/integration/test_cases/inference/completion.json b/tests/integration/test_cases/inference/completion.json index 731ceddbc..447087d19 100644 --- a/tests/integration/test_cases/inference/completion.json +++ b/tests/integration/test_cases/inference/completion.json @@ -40,8 +40,8 @@ "user_input": "Michael Jordan was born in 1963. He played basketball for the Chicago Bulls. He retired in 2003.", "expected": { "name": "Michael Jordan", - "year_born": "1963", - "year_retired": "2003" + "year_born": 1963, + "year_retired": 2003 } } }, From 6377b1912b58d92438311cfc9547c1bb2b6de9b3 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Mon, 24 Mar 2025 18:54:52 -0400 Subject: [PATCH 11/15] Revert "Use int for year in test case" This reverts commit 5426341d5be7e00d95a933c0cae24715bd2436d2. --- tests/integration/inference/test_text_inference.py | 4 ++-- tests/integration/test_cases/inference/completion.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/inference/test_text_inference.py b/tests/integration/inference/test_text_inference.py index 5410f20c5..a3cfce4fd 100644 --- a/tests/integration/inference/test_text_inference.py +++ b/tests/integration/inference/test_text_inference.py @@ -199,8 +199,8 @@ def test_text_completion_structured_output(client_with_models, text_model_id, te class AnswerFormat(BaseModel): name: str - year_born: int - year_retired: int + year_born: str + year_retired: str tc = TestCase(test_case) diff --git a/tests/integration/test_cases/inference/completion.json b/tests/integration/test_cases/inference/completion.json index 447087d19..731ceddbc 100644 --- a/tests/integration/test_cases/inference/completion.json +++ b/tests/integration/test_cases/inference/completion.json @@ -40,8 +40,8 @@ "user_input": "Michael Jordan was born in 1963. He played basketball for the Chicago Bulls. He retired in 2003.", "expected": { "name": "Michael Jordan", - "year_born": 1963, - "year_retired": 2003 + "year_born": "1963", + "year_retired": "2003" } } }, From 6135bdec229636102eacaa15aa4c35181fde16c9 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Tue, 22 Apr 2025 13:09:36 -0400 Subject: [PATCH 12/15] add tests/verification/conf/lmstudio.yaml --- tests/verifications/conf/lmstudio.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/verifications/conf/lmstudio.yaml diff --git a/tests/verifications/conf/lmstudio.yaml b/tests/verifications/conf/lmstudio.yaml new file mode 100644 index 000000000..40d40d6d5 --- /dev/null +++ b/tests/verifications/conf/lmstudio.yaml @@ -0,0 +1,16 @@ +base_url: http://localhost:1234/v1/ +models: +- meta-llama-3.1-8b-instruct +- llama-3.2-3b-instruct +model_display_names: + meta-llama-3.1-8b-instruct: Llama-3.1-8b-Instruct + llama-3.2-3b-instruct: Llama-3.2-3b-Instruct +test_exclusions: + meta-llama-3.1-8b-instruct: + - test_chat_non_streaming_image + - test_chat_streaming_image + - test_chat_multi_turn_multiple_images + llama-3.2-3b-instruct: + - test_chat_non_streaming_image + - test_chat_streaming_image + - test_chat_multi_turn_multiple_images From cfc6bdae680f74b36254f815d301907ac6d5d3f2 Mon Sep 17 00:00:00 2001 From: Matt Clayton Date: Fri, 25 Apr 2025 13:38:40 -0400 Subject: [PATCH 13/15] llama-4-scout-17b-16e-instruct passing tests --- tests/verifications/REPORT.md | 46 + tests/verifications/conf/lmstudio.yaml | 12 +- .../verifications/test_results/lmstudio.json | 1101 +++++++++++++++++ 3 files changed, 1150 insertions(+), 9 deletions(-) create mode 100644 tests/verifications/test_results/lmstudio.json diff --git a/tests/verifications/REPORT.md b/tests/verifications/REPORT.md index 2a700fa9c..674b6ad15 100644 --- a/tests/verifications/REPORT.md +++ b/tests/verifications/REPORT.md @@ -19,6 +19,7 @@ | Together | 50.0% | 40 | 80 | | Fireworks | 50.0% | 40 | 80 | | Openai | 100.0% | 56 | 56 | +| Lmstudio | 100.0% | 24 | 24 | @@ -230,3 +231,48 @@ pytest tests/verifications/openai_api/test_chat_completion.py --provider=openai | test_chat_streaming_tool_calling | ✅ | ✅ | | test_chat_streaming_tool_choice_none | ✅ | ✅ | | test_chat_streaming_tool_choice_required | ✅ | ✅ | + +## Lmstudio + +```bash +# Run all tests for this provider: +pytest tests/verifications/openai_api/test_chat_completion.py --provider=lmstudio -v + +# Example: Run only the 'earth' case of test_chat_non_streaming_basic: +pytest tests/verifications/openai_api/test_chat_completion.py --provider=lmstudio -k "test_chat_non_streaming_basic and earth" +``` + + +**Model Key (Lmstudio)** + +| Display Name | Full Model ID | +| --- | --- | +| Llama-4-Scout-Instruct | `llama-4-scout-17b-16e-instruct` | + + +| Test | Llama-4-Scout-Instruct | +| --- | --- | +| test_chat_non_streaming_basic (earth) | ✅ | +| test_chat_non_streaming_basic (saturn) | ✅ | +| test_chat_non_streaming_multi_turn_tool_calling (add_product_tool) | ✅ | +| test_chat_non_streaming_multi_turn_tool_calling (compare_monthly_expense_tool) | ✅ | +| test_chat_non_streaming_multi_turn_tool_calling (get_then_create_event_tool) | ✅ | +| test_chat_non_streaming_multi_turn_tool_calling (text_then_weather_tool) | ✅ | +| test_chat_non_streaming_multi_turn_tool_calling (weather_tool_then_text) | ✅ | +| test_chat_non_streaming_structured_output (calendar) | ✅ | +| test_chat_non_streaming_structured_output (math) | ✅ | +| test_chat_non_streaming_tool_calling | ✅ | +| test_chat_non_streaming_tool_choice_none | ✅ | +| test_chat_non_streaming_tool_choice_required | ✅ | +| test_chat_streaming_basic (earth) | ✅ | +| test_chat_streaming_basic (saturn) | ✅ | +| test_chat_streaming_multi_turn_tool_calling (add_product_tool) | ✅ | +| test_chat_streaming_multi_turn_tool_calling (compare_monthly_expense_tool) | ✅ | +| test_chat_streaming_multi_turn_tool_calling (get_then_create_event_tool) | ✅ | +| test_chat_streaming_multi_turn_tool_calling (text_then_weather_tool) | ✅ | +| test_chat_streaming_multi_turn_tool_calling (weather_tool_then_text) | ✅ | +| test_chat_streaming_structured_output (calendar) | ✅ | +| test_chat_streaming_structured_output (math) | ✅ | +| test_chat_streaming_tool_calling | ✅ | +| test_chat_streaming_tool_choice_none | ✅ | +| test_chat_streaming_tool_choice_required | ✅ | diff --git a/tests/verifications/conf/lmstudio.yaml b/tests/verifications/conf/lmstudio.yaml index 40d40d6d5..c31b4273c 100644 --- a/tests/verifications/conf/lmstudio.yaml +++ b/tests/verifications/conf/lmstudio.yaml @@ -1,16 +1,10 @@ base_url: http://localhost:1234/v1/ models: -- meta-llama-3.1-8b-instruct -- llama-3.2-3b-instruct +- llama-4-scout-17b-16e-instruct model_display_names: - meta-llama-3.1-8b-instruct: Llama-3.1-8b-Instruct - llama-3.2-3b-instruct: Llama-3.2-3b-Instruct + llama-4-scout-17b-16e-instruct: Llama-4-Scout-Instruct test_exclusions: - meta-llama-3.1-8b-instruct: - - test_chat_non_streaming_image - - test_chat_streaming_image - - test_chat_multi_turn_multiple_images - llama-3.2-3b-instruct: + llama-4-scout-17b-16e-instruct: - test_chat_non_streaming_image - test_chat_streaming_image - test_chat_multi_turn_multiple_images diff --git a/tests/verifications/test_results/lmstudio.json b/tests/verifications/test_results/lmstudio.json new file mode 100644 index 000000000..607606365 --- /dev/null +++ b/tests/verifications/test_results/lmstudio.json @@ -0,0 +1,1101 @@ +{ + "created": 1745602243.167073, + "duration": 116.09479594230652, + "exitcode": 0, + "root": "/Users/macstudio1lmstudio/Projects/llama-stack", + "environment": {}, + "summary": { + "passed": 24, + "skipped": 4, + "total": 28, + "collected": 28 + }, + "collectors": [ + { + "nodeid": "", + "outcome": "passed", + "result": [ + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py", + "type": "Module" + } + ] + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py", + "outcome": "passed", + "result": [ + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[llama-4-scout-17b-16e-instruct-earth]", + "type": "Function", + "lineno": 95 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[llama-4-scout-17b-16e-instruct-saturn]", + "type": "Function", + "lineno": 95 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[llama-4-scout-17b-16e-instruct-earth]", + "type": "Function", + "lineno": 114 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[llama-4-scout-17b-16e-instruct-saturn]", + "type": "Function", + "lineno": 114 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 138 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 157 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[llama-4-scout-17b-16e-instruct-calendar]", + "type": "Function", + "lineno": 181 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[llama-4-scout-17b-16e-instruct-math]", + "type": "Function", + "lineno": 181 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[llama-4-scout-17b-16e-instruct-calendar]", + "type": "Function", + "lineno": 204 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[llama-4-scout-17b-16e-instruct-math]", + "type": "Function", + "lineno": 204 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 226 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 250 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 278 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 302 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 329 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[llama-4-scout-17b-16e-instruct-case0]", + "type": "Function", + "lineno": 352 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-text_then_weather_tool]", + "type": "Function", + "lineno": 395 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-weather_tool_then_text]", + "type": "Function", + "lineno": 395 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-add_product_tool]", + "type": "Function", + "lineno": 395 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-get_then_create_event_tool]", + "type": "Function", + "lineno": 395 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool]", + "type": "Function", + "lineno": 395 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-text_then_weather_tool]", + "type": "Function", + "lineno": 526 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-weather_tool_then_text]", + "type": "Function", + "lineno": 526 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-add_product_tool]", + "type": "Function", + "lineno": 526 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-get_then_create_event_tool]", + "type": "Function", + "lineno": 526 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool]", + "type": "Function", + "lineno": 526 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[llama-4-scout-17b-16e-instruct-stream=False]", + "type": "Function", + "lineno": 609 + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[llama-4-scout-17b-16e-instruct-stream=True]", + "type": "Function", + "lineno": 609 + } + ] + } + ], + "tests": [ + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[llama-4-scout-17b-16e-instruct-earth]", + "lineno": 95, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_basic[llama-4-scout-17b-16e-instruct-earth]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-earth", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "earth" + }, + "setup": { + "duration": 0.028099916000428493, + "outcome": "passed" + }, + "call": { + "duration": 2.1059866249997867, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00014304199976322707, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[llama-4-scout-17b-16e-instruct-saturn]", + "lineno": 95, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_basic[llama-4-scout-17b-16e-instruct-saturn]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-saturn", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "saturn" + }, + "setup": { + "duration": 0.009213250001266715, + "outcome": "passed" + }, + "call": { + "duration": 1.321610500001043, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00012754199997289106, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[llama-4-scout-17b-16e-instruct-earth]", + "lineno": 114, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_basic[llama-4-scout-17b-16e-instruct-earth]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-earth", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "earth" + }, + "setup": { + "duration": 0.006229208000149811, + "outcome": "passed" + }, + "call": { + "duration": 0.3756380410013662, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00011541699859662913, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[llama-4-scout-17b-16e-instruct-saturn]", + "lineno": 114, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_basic[llama-4-scout-17b-16e-instruct-saturn]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-saturn", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "saturn" + }, + "setup": { + "duration": 0.0063281250004365575, + "outcome": "passed" + }, + "call": { + "duration": 1.316346125000564, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00012770799912686925, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 138, + "outcome": "skipped", + "keywords": [ + "test_chat_non_streaming_image[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.006362333000652143, + "outcome": "passed" + }, + "call": { + "duration": 0.00012162500024714973, + "outcome": "skipped", + "longrepr": "('/Users/macstudio1lmstudio/Projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 147, 'Skipped: Skipping test_chat_non_streaming_image for model llama-4-scout-17b-16e-instruct on provider lmstudio based on config.')" + }, + "teardown": { + "duration": 0.00008449999950244091, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 157, + "outcome": "skipped", + "keywords": [ + "test_chat_streaming_image[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.0059984159997839015, + "outcome": "passed" + }, + "call": { + "duration": 0.00011524999899847899, + "outcome": "skipped", + "longrepr": "('/Users/macstudio1lmstudio/Projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 166, 'Skipped: Skipping test_chat_streaming_image for model llama-4-scout-17b-16e-instruct on provider lmstudio based on config.')" + }, + "teardown": { + "duration": 0.0000853750007081544, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[llama-4-scout-17b-16e-instruct-calendar]", + "lineno": 181, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_structured_output[llama-4-scout-17b-16e-instruct-calendar]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-calendar", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "calendar" + }, + "setup": { + "duration": 0.009981625000364147, + "outcome": "passed" + }, + "call": { + "duration": 1.3905797079987678, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0001315829995292006, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[llama-4-scout-17b-16e-instruct-math]", + "lineno": 181, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_structured_output[llama-4-scout-17b-16e-instruct-math]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-math", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "math" + }, + "setup": { + "duration": 0.005977208000331302, + "outcome": "passed" + }, + "call": { + "duration": 9.832755792000171, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010983399988617748, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[llama-4-scout-17b-16e-instruct-calendar]", + "lineno": 204, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_structured_output[llama-4-scout-17b-16e-instruct-calendar]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-calendar", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "calendar" + }, + "setup": { + "duration": 0.00787095799933013, + "outcome": "passed" + }, + "call": { + "duration": 1.3666670000002341, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010829199891304597, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[llama-4-scout-17b-16e-instruct-math]", + "lineno": 204, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_structured_output[llama-4-scout-17b-16e-instruct-math]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-math", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "math" + }, + "setup": { + "duration": 0.006011375000525732, + "outcome": "passed" + }, + "call": { + "duration": 9.814112499998373, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010850000035134144, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 226, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_tool_calling[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.006303957999989507, + "outcome": "passed" + }, + "call": { + "duration": 2.3944496660005825, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010325000039301813, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 250, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_tool_calling[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.005938166999840178, + "outcome": "passed" + }, + "call": { + "duration": 0.7450492919997487, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010945800022454932, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 278, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_tool_choice_required[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.005958375000773231, + "outcome": "passed" + }, + "call": { + "duration": 0.7705123750001803, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010650000149325933, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 302, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_tool_choice_required[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.00633945800109359, + "outcome": "passed" + }, + "call": { + "duration": 0.7685649579998426, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010245799967378844, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 329, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_tool_choice_none[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.0064487090003240155, + "outcome": "passed" + }, + "call": { + "duration": 17.334407000000283, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00011550000090210233, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[llama-4-scout-17b-16e-instruct-case0]", + "lineno": 352, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_tool_choice_none[llama-4-scout-17b-16e-instruct-case0]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-case0", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "case0" + }, + "setup": { + "duration": 0.008446583000477403, + "outcome": "passed" + }, + "call": { + "duration": 16.891984292000416, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00011674999950628262, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-text_then_weather_tool]", + "lineno": 395, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-text_then_weather_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-text_then_weather_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "text_then_weather_tool" + }, + "setup": { + "duration": 0.013226832999862381, + "outcome": "passed" + }, + "call": { + "duration": 4.760952832999465, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0001083329989342019, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-weather_tool_then_text]", + "lineno": 395, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-weather_tool_then_text]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-weather_tool_then_text", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "weather_tool_then_text" + }, + "setup": { + "duration": 0.006158666999908746, + "outcome": "passed" + }, + "call": { + "duration": 1.864827041999888, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010883399954764172, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-add_product_tool]", + "lineno": 395, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-add_product_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-add_product_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "add_product_tool" + }, + "setup": { + "duration": 0.006072582998967846, + "outcome": "passed" + }, + "call": { + "duration": 4.076500666998982, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00011045800056308508, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-get_then_create_event_tool]", + "lineno": 395, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-get_then_create_event_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-get_then_create_event_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "get_then_create_event_tool" + }, + "setup": { + "duration": 0.00609904200064193, + "outcome": "passed" + }, + "call": { + "duration": 9.440772791998825, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0001123750007536728, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool]", + "lineno": 395, + "outcome": "passed", + "keywords": [ + "test_chat_non_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "compare_monthly_expense_tool" + }, + "setup": { + "duration": 0.005757832999734092, + "outcome": "passed" + }, + "call": { + "duration": 4.545131082999433, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010958299935737159, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-text_then_weather_tool]", + "lineno": 526, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-text_then_weather_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-text_then_weather_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "text_then_weather_tool" + }, + "setup": { + "duration": 0.006187499999214197, + "outcome": "passed" + }, + "call": { + "duration": 4.744507708001038, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010883299910346977, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-weather_tool_then_text]", + "lineno": 526, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-weather_tool_then_text]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-weather_tool_then_text", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "weather_tool_then_text" + }, + "setup": { + "duration": 0.006288458998824353, + "outcome": "passed" + }, + "call": { + "duration": 1.8597102080002514, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0001077090000762837, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-add_product_tool]", + "lineno": 526, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-add_product_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-add_product_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "add_product_tool" + }, + "setup": { + "duration": 0.0060759169991797535, + "outcome": "passed" + }, + "call": { + "duration": 4.066417875001207, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010712499897636008, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-get_then_create_event_tool]", + "lineno": 526, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-get_then_create_event_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-get_then_create_event_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "get_then_create_event_tool" + }, + "setup": { + "duration": 0.006023833000654122, + "outcome": "passed" + }, + "call": { + "duration": 9.450671958000385, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00010741699952632189, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool]", + "lineno": 526, + "outcome": "passed", + "keywords": [ + "test_chat_streaming_multi_turn_tool_calling[llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-compare_monthly_expense_tool", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "compare_monthly_expense_tool" + }, + "setup": { + "duration": 0.005968625000605243, + "outcome": "passed" + }, + "call": { + "duration": 4.545033249998596, + "outcome": "passed" + }, + "teardown": { + "duration": 0.000334707998263184, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[llama-4-scout-17b-16e-instruct-stream=False]", + "lineno": 609, + "outcome": "skipped", + "keywords": [ + "test_chat_multi_turn_multiple_images[llama-4-scout-17b-16e-instruct-stream=False]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-stream=False", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "stream=False" + }, + "setup": { + "duration": 0.007036916000288329, + "outcome": "passed" + }, + "call": { + "duration": 0.00010395800018159207, + "outcome": "skipped", + "longrepr": "('/Users/macstudio1lmstudio/Projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 616, 'Skipped: Skipping test_chat_multi_turn_multiple_images for model llama-4-scout-17b-16e-instruct on provider lmstudio based on config.')" + }, + "teardown": { + "duration": 0.00007462500070687383, + "outcome": "passed" + } + }, + { + "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[llama-4-scout-17b-16e-instruct-stream=True]", + "lineno": 609, + "outcome": "skipped", + "keywords": [ + "test_chat_multi_turn_multiple_images[llama-4-scout-17b-16e-instruct-stream=True]", + "parametrize", + "pytestmark", + "llama-4-scout-17b-16e-instruct-stream=True", + "test_chat_completion.py", + "openai_api", + "verifications", + "tests", + "llama-stack", + "" + ], + "metadata": { + "model": "llama-4-scout-17b-16e-instruct", + "case_id": "stream=True" + }, + "setup": { + "duration": 0.006525624999994761, + "outcome": "passed" + }, + "call": { + "duration": 0.00008283300121547654, + "outcome": "skipped", + "longrepr": "('/Users/macstudio1lmstudio/Projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 616, 'Skipped: Skipping test_chat_multi_turn_multiple_images for model llama-4-scout-17b-16e-instruct on provider lmstudio based on config.')" + }, + "teardown": { + "duration": 0.000543541998922592, + "outcome": "passed" + } + } + ] +} From 59e1c5f4a0b3a63ec910ae9896f68fd41a6df6ab Mon Sep 17 00:00:00 2001 From: Matt Clayton Date: Sun, 27 Apr 2025 15:24:37 -0400 Subject: [PATCH 14/15] Pass 1 for pre-commit fixes --- .../self_hosted_distro/lmstudio.md | 2 +- .../remote/inference/lmstudio/_client.py | 161 ++++++++---------- .../remote/inference/lmstudio/lmstudio.py | 50 ++++-- .../remote/inference/lmstudio/models.py | 4 +- llama_stack/templates/dependencies.json | 4 + llama_stack/templates/lmstudio/build.yaml | 1 - llama_stack/templates/lmstudio/run.yaml | 6 +- 7 files changed, 119 insertions(+), 109 deletions(-) diff --git a/docs/source/distributions/self_hosted_distro/lmstudio.md b/docs/source/distributions/self_hosted_distro/lmstudio.md index 800a4af66..e96e88aab 100644 --- a/docs/source/distributions/self_hosted_distro/lmstudio.md +++ b/docs/source/distributions/self_hosted_distro/lmstudio.md @@ -33,7 +33,7 @@ The following models are available by default: - `meta-llama-3.1-70b-instruct ` - `llama-3.2-1b-instruct ` - `llama-3.2-3b-instruct ` -- `llama-3.2-70b-instruct ` +- `llama-3.3-70b-instruct ` - `nomic-embed-text-v1.5 ` - `all-minilm-l6-v2 ` diff --git a/llama_stack/providers/remote/inference/lmstudio/_client.py b/llama_stack/providers/remote/inference/lmstudio/_client.py index 4c4df3717..5b2e2c738 100644 --- a/llama_stack/providers/remote/inference/lmstudio/_client.py +++ b/llama_stack/providers/remote/inference/lmstudio/_client.py @@ -1,13 +1,20 @@ -import asyncio -from typing import AsyncIterator, AsyncGenerator, List, Literal, Optional, Union -import lmstudio as lms -import json -import re -import logging +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. +import asyncio +import json +import logging +import re +from typing import Any, AsyncIterator, List, Literal, Optional, Union + +import lmstudio as lms +from openai import AsyncOpenAI as OpenAI from llama_stack.apis.common.content_types import InterleavedContent, TextDelta -from llama_stack.apis.inference.inference import ( +from llama_stack.apis.inference import ( ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseEvent, @@ -16,15 +23,14 @@ from llama_stack.apis.inference.inference import ( CompletionMessage, CompletionResponse, CompletionResponseStreamChunk, + GrammarResponseFormat, + GreedySamplingStrategy, JsonSchemaResponseFormat, Message, - ToolConfig, - ToolDefinition, -) -from llama_stack.models.llama.datatypes import ( - GreedySamplingStrategy, SamplingParams, StopReason, + ToolConfig, + ToolDefinition, TopKSamplingStrategy, TopPSamplingStrategy, ) @@ -38,7 +44,6 @@ from llama_stack.providers.utils.inference.prompt_adapter import ( content_has_media, interleaved_content_as_str, ) -from openai import AsyncOpenAI as OpenAI LlmPredictionStopReason = Literal[ "userStopped", @@ -57,13 +62,15 @@ class LMStudioClient: self.url = url self.sdk_client = lms.Client(self.url) self.openai_client = OpenAI(base_url=f"http://{url}/v1", api_key="lmstudio") - + # Standard error handling helper methods def _log_error(self, error, context=""): """Centralized error logging method""" logging.warning(f"Error in LMStudio {context}: {error}") - - async def _create_fallback_chat_stream(self, error_message="I encountered an error processing your request."): + + async def _create_fallback_chat_stream( + self, error_message="I encountered an error processing your request." + ) -> AsyncIterator[ChatCompletionResponseStreamChunk]: """Create a standardized fallback stream for chat completions""" yield ChatCompletionResponseStreamChunk( event=ChatCompletionResponseEvent( @@ -83,30 +90,32 @@ class LMStudioClient: delta=TextDelta(text=""), ) ) - + async def _create_fallback_completion_stream(self, error_message="Error processing response"): """Create a standardized fallback stream for text completions""" yield CompletionResponseStreamChunk( delta=error_message, ) - - def _create_fallback_chat_response(self, error_message="I encountered an error processing your request."): + + def _create_fallback_chat_response( + self, error_message="I encountered an error processing your request." + ) -> ChatCompletionResponse: """Create a standardized fallback response for chat completions""" return ChatCompletionResponse( - message=Message( + completion_message=CompletionMessage( role="assistant", content=error_message, - ), - stop_reason=StopReason.end_of_message, + stop_reason=StopReason.end_of_message, + ) ) - - def _create_fallback_completion_response(self, error_message="Error processing response"): + + def _create_fallback_completion_response(self, error_message="Error processing response") -> CompletionResponse: """Create a standardized fallback response for text completions""" return CompletionResponse( content=error_message, stop_reason=StopReason.end_of_message, ) - + def _handle_json_extraction(self, content, context="JSON extraction"): """Standardized method to extract valid JSON from potentially malformed content""" try: @@ -114,14 +123,14 @@ class LMStudioClient: return json.dumps(json_content) # Re-serialize to ensure valid JSON except json.JSONDecodeError as e: self._log_error(e, f"{context} - Attempting to extract valid JSON") - + json_patterns = [ - r'(\{.*\})', # Match anything between curly braces - r'(\[.*\])', # Match anything between square brackets - r'```json\s*([\s\S]*?)\s*```', # Match content in JSON code blocks - r'```\s*([\s\S]*?)\s*```', # Match content in any code blocks + r"(\{.*\})", # Match anything between curly braces + r"(\[.*\])", # Match anything between square brackets + r"```json\s*([\s\S]*?)\s*```", # Match content in JSON code blocks + r"```\s*([\s\S]*?)\s*```", # Match content in any code blocks ] - + for pattern in json_patterns: json_match = re.search(pattern, content, re.DOTALL) if json_match: @@ -131,7 +140,7 @@ class LMStudioClient: return json.dumps(json_content) # Re-serialize to ensure valid JSON except json.JSONDecodeError: continue # Try the next pattern - + # If we couldn't extract valid JSON, log a warning self._log_error("Failed to extract valid JSON", context) return None @@ -148,14 +157,10 @@ class LMStudioClient: return False async def get_embedding_model(self, provider_model_id: str): - model = await asyncio.to_thread( - self.sdk_client.embedding.model, provider_model_id - ) + model = await asyncio.to_thread(self.sdk_client.embedding.model, provider_model_id) return model - async def embed( - self, embedding_model: lms.EmbeddingModel, contents: Union[str, List[str]] - ): + async def embed(self, embedding_model: lms.EmbeddingModel, contents: Union[str, List[str]]): embeddings = await asyncio.to_thread(embedding_model.embed, contents) return embeddings @@ -170,14 +175,12 @@ class LMStudioClient: sampling_params: Optional[SamplingParams] = None, json_schema: Optional[JsonSchemaResponseFormat] = None, stream: Optional[bool] = False, - ) -> Union[ - ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] - ]: + ) -> Union[ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk]]: chat = self._convert_message_list_to_lmstudio_chat(messages) config = self._get_completion_config_from_params(sampling_params) if stream: - async def stream_generator(): + async def stream_generator() -> AsyncIterator[ChatCompletionResponseStreamChunk]: prediction_stream = await asyncio.to_thread( llm.respond_stream, history=chat, @@ -191,7 +194,7 @@ class LMStudioClient: delta=TextDelta(text=""), ) ) - + async for chunk in self._async_iterate(prediction_stream): yield ChatCompletionResponseStreamChunk( event=ChatCompletionResponseEvent( @@ -225,9 +228,7 @@ class LMStudioClient: stream: Optional[bool] = False, tools: Optional[List[ToolDefinition]] = None, tool_config: Optional[ToolConfig] = None, - ) -> Union[ - ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] - ]: + ) -> Union[ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk]]: try: model_key = llm.get_info().model_key request = ChatCompletionRequest( @@ -240,17 +241,15 @@ class LMStudioClient: stream=stream, ) rest_request = await self._convert_request_to_rest_call(request) - + if stream: try: stream = await self.openai_client.chat.completions.create(**rest_request) - return convert_openai_chat_completion_stream( - stream, enable_incremental_tool_calls=True - ) + return convert_openai_chat_completion_stream(stream, enable_incremental_tool_calls=True) except Exception as e: self._log_error(e, "streaming tool calling") return self._create_fallback_chat_stream() - + try: response = await self.openai_client.chat.completions.create(**rest_request) if response: @@ -280,9 +279,7 @@ class LMStudioClient: stream: Optional[bool] = False, tools: Optional[List[ToolDefinition]] = None, tool_config: Optional[ToolConfig] = None, - ) -> Union[ - ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] - ]: + ) -> Union[ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk]]: if tools is None or len(tools) == 0: return await self._llm_respond_non_tools( llm=llm, @@ -313,7 +310,7 @@ class LMStudioClient: config = self._get_completion_config_from_params(sampling_params) if stream: - async def stream_generator(): + async def stream_generator() -> AsyncIterator[CompletionResponseStreamChunk]: try: prediction_stream = await asyncio.to_thread( llm.complete_stream, @@ -341,7 +338,7 @@ class LMStudioClient: config=config, response_format=json_schema, ) - + # If we have a JSON schema, ensure the response is valid JSON if json_schema is not None: valid_json = self._handle_json_extraction(response.content, "completion response") @@ -351,7 +348,7 @@ class LMStudioClient: stop_reason=self._get_stop_reason(response.stats.stop_reason), ) # If we couldn't extract valid JSON, continue with the original content - + return CompletionResponse( content=response.content, stop_reason=self._get_stop_reason(response.stats.stop_reason), @@ -361,15 +358,11 @@ class LMStudioClient: # Return a fallback response with an error message return self._create_fallback_completion_response() - def _convert_message_list_to_lmstudio_chat( - self, messages: List[Message] - ) -> lms.Chat: + def _convert_message_list_to_lmstudio_chat(self, messages: List[Message]) -> lms.Chat: chat = lms.Chat() for message in messages: if content_has_media(message.content): - raise NotImplementedError( - "Media content is not supported in LMStudio messages" - ) + raise NotImplementedError("Media content is not supported in LMStudio messages") if message.role == "user": chat.add_user_message(interleaved_content_as_str(message.content)) elif message.role == "system": @@ -380,9 +373,7 @@ class LMStudioClient: raise ValueError(f"Unsupported message role: {message.role}") return chat - def _convert_prediction_to_chat_response( - self, result: lms.PredictionResult - ) -> ChatCompletionResponse: + def _convert_prediction_to_chat_response(self, result: lms.PredictionResult) -> ChatCompletionResponse: response = ChatCompletionResponse( completion_message=CompletionMessage( content=result.content, @@ -415,11 +406,7 @@ class LMStudioClient: options.update( { "maxTokens": params.max_tokens if params.max_tokens != 0 else None, - "repetitionPenalty": ( - params.repetition_penalty - if params.repetition_penalty != 0 - else None - ), + "repetitionPenalty": (params.repetition_penalty if params.repetition_penalty != 0 else None), } ) return options @@ -449,32 +436,30 @@ class LMStudioClient: break yield item - async def _convert_request_to_rest_call( - self, request: ChatCompletionRequest - ) -> dict: + async def _convert_request_to_rest_call(self, request: ChatCompletionRequest) -> dict: compatible_request = self._convert_sampling_params(request.sampling_params) compatible_request["model"] = request.model - compatible_request["messages"] = [ - await convert_message_to_openai_dict_new(m) for m in request.messages - ] + compatible_request["messages"] = [await convert_message_to_openai_dict_new(m) for m in request.messages] if request.response_format: - compatible_request["response_format"] = { - "type": "json_schema", - "json_schema": request.response_format.json_schema, - } + if isinstance(request.response_format, JsonSchemaResponseFormat): + compatible_request["response_format"] = { + "type": "json_schema", + "json_schema": request.response_format.json_schema, + } + elif isinstance(request.response_format, GrammarResponseFormat): + compatible_request["response_format"] = { + "type": "grammar", + "bnf": request.response_format.bnf, + } if request.tools is not None: - compatible_request["tools"] = [ - convert_tooldef_to_openai_tool(tool) for tool in request.tools - ] + compatible_request["tools"] = [convert_tooldef_to_openai_tool(tool) for tool in request.tools] compatible_request["logprobs"] = False compatible_request["stream"] = request.stream - compatible_request["extra_headers"] = { - b"User-Agent": b"llama-stack: lmstudio-inference-adapter" - } + compatible_request["extra_headers"] = {b"User-Agent": b"llama-stack: lmstudio-inference-adapter"} return compatible_request def _convert_sampling_params(self, sampling_params: Optional[SamplingParams]) -> dict: - params = {} + params: dict[str, Any] = {} if sampling_params is None: return params diff --git a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py index 35380a58b..8f7377bc9 100644 --- a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py +++ b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py @@ -14,7 +14,9 @@ from llama_stack.apis.inference import ( ChatCompletionResponse, EmbeddingsResponse, EmbeddingTaskType, + GrammarResponseFormat, Inference, + JsonSchemaResponseFormat, LogProbConfig, Message, ResponseFormat, @@ -29,7 +31,6 @@ from llama_stack.apis.inference.inference import ( ChatCompletionResponseStreamChunk, CompletionResponse, CompletionResponseStreamChunk, - ResponseFormatType, ) from llama_stack.providers.datatypes import ModelsProtocolPrivate from llama_stack.providers.remote.inference.lmstudio._client import LMStudioClient @@ -50,6 +51,18 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): def client(self) -> LMStudioClient: return LMStudioClient(url=self.url) + async def batch_chat_completion(self, *args, **kwargs): + raise NotImplementedError("Batch chat completion not supported by LM Studio Provider") + + async def batch_completion(self, *args, **kwargs): + raise NotImplementedError("Batch completion not supported by LM Studio Provider") + + async def openai_chat_completion(self, *args, **kwargs): + raise NotImplementedError("OpenAI chat completion not supported by LM Studio Provider") + + async def openai_completion(self, *args, **kwargs): + raise NotImplementedError("OpenAI completion not supported by LM Studio Provider") + async def initialize(self) -> None: pass @@ -71,9 +84,12 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): assert all(not content_has_media(content) for content in contents), ( "Media content not supported in embedding model" ) + if self.model_store is None: + raise ValueError("ModelStore is not initialized") model = await self.model_store.get_model(model_id) embedding_model = await self.client.get_embedding_model(model.provider_model_id) - embeddings = await self.client.embed(embedding_model, contents) + string_contents = [item.text if hasattr(item, "text") else str(item) for item in contents] + embeddings = await self.client.embed(embedding_model, string_contents) return EmbeddingsResponse(embeddings=embeddings) async def chat_completion( @@ -81,26 +97,31 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): model_id: str, messages: List[Message], sampling_params: Optional[SamplingParams] = None, - response_format: Optional[ResponseFormat] = None, tools: Optional[List[ToolDefinition]] = None, - tool_choice: Optional[ToolChoice] = ToolChoice.auto, + tool_choice: Optional[ToolChoice] = None, # Default value changed from ToolChoice.auto to None tool_prompt_format: Optional[ToolPromptFormat] = None, + response_format: Optional[ + Union[JsonSchemaResponseFormat, GrammarResponseFormat] + ] = None, # Moved and type changed stream: Optional[bool] = False, logprobs: Optional[LogProbConfig] = None, tool_config: Optional[ToolConfig] = None, ) -> Union[ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk]]: + if self.model_store is None: + raise ValueError("ModelStore is not initialized") model = await self.model_store.get_model(model_id) llm = await self.client.get_llm(model.provider_model_id) - if response_format is not None and response_format.type != ResponseFormatType.json_schema.value: - raise ValueError(f"Response format type {response_format.type} not supported for LM Studio Provider") - json_schema = response_format.json_schema if response_format else None - + json_schema_format = response_format if isinstance(response_format, JsonSchemaResponseFormat) else None + if response_format is not None and not isinstance(response_format, JsonSchemaResponseFormat): + raise ValueError( + f"Response format type {type(response_format).__name__} not supported for LM Studio Provider" + ) return await self.client.llm_respond( llm=llm, messages=messages, sampling_params=sampling_params, - json_schema=json_schema, + json_schema=json_schema_format, stream=stream, tool_config=tool_config, tools=tools, @@ -115,13 +136,16 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): stream: Optional[bool] = False, logprobs: Optional[LogProbConfig] = None, # Skip this for now ) -> Union[CompletionResponse, AsyncIterator[CompletionResponseStreamChunk]]: + if self.model_store is None: + raise ValueError("ModelStore is not initialized") model = await self.model_store.get_model(model_id) llm = await self.client.get_llm(model.provider_model_id) if content_has_media(content): raise NotImplementedError("Media content not supported in LM Studio Provider") - if response_format is not None and response_format.type != ResponseFormatType.json_schema.value: - raise ValueError(f"Response format type {response_format.type} not supported for LM Studio Provider") - json_schema = response_format.json_schema if response_format else None + if not isinstance(response_format, JsonSchemaResponseFormat): + raise ValueError( + f"Response format type {type(response_format).__name__} not supported for LM Studio Provider" + ) - return await self.client.llm_completion(llm, content, sampling_params, json_schema, stream) + return await self.client.llm_completion(llm, content, sampling_params, response_format, stream) diff --git a/llama_stack/providers/remote/inference/lmstudio/models.py b/llama_stack/providers/remote/inference/lmstudio/models.py index 55524aea2..e98c6d609 100644 --- a/llama_stack/providers/remote/inference/lmstudio/models.py +++ b/llama_stack/providers/remote/inference/lmstudio/models.py @@ -5,7 +5,7 @@ # the root directory of this source tree. from llama_stack.apis.models.models import ModelType -from llama_stack.models.llama.datatypes import CoreModelId +from llama_stack.models.llama.sku_list import CoreModelId from llama_stack.providers.utils.inference.model_registry import ( ProviderModelEntry, ) @@ -63,9 +63,7 @@ MODEL_ENTRIES = [ }, ), ProviderModelEntry( - model_id="all-MiniLM-L6-v2", provider_model_id="all-minilm-l6-v2", - provider_id="lmstudio", model_type=ModelType.embedding, metadata={ "embedding_dimension": 384, diff --git a/llama_stack/templates/dependencies.json b/llama_stack/templates/dependencies.json index 83864a6fc..cc564e2e5 100644 --- a/llama_stack/templates/dependencies.json +++ b/llama_stack/templates/dependencies.json @@ -351,10 +351,12 @@ "chardet", "chromadb-client", "datasets", + "emoji", "faiss-cpu", "fastapi", "fire", "httpx", + "langdetect", "lmstudio", "matplotlib", "nltk", @@ -367,6 +369,7 @@ "psycopg2-binary", "pymongo", "pypdf", + "pythainlp", "redis", "requests", "scikit-learn", @@ -374,6 +377,7 @@ "sentencepiece", "tqdm", "transformers", + "tree_sitter", "uvicorn" ], "meta-reference-gpu": [ diff --git a/llama_stack/templates/lmstudio/build.yaml b/llama_stack/templates/lmstudio/build.yaml index 6fd713766..5167af2c3 100644 --- a/llama_stack/templates/lmstudio/build.yaml +++ b/llama_stack/templates/lmstudio/build.yaml @@ -24,7 +24,6 @@ distribution_spec: telemetry: - inline::meta-reference tool_runtime: - - remote::brave-search - remote::tavily-search - inline::code-interpreter - inline::rag-runtime diff --git a/llama_stack/templates/lmstudio/run.yaml b/llama_stack/templates/lmstudio/run.yaml index b23ed6da9..ac3b031d7 100644 --- a/llama_stack/templates/lmstudio/run.yaml +++ b/llama_stack/templates/lmstudio/run.yaml @@ -75,7 +75,7 @@ providers: - provider_id: meta-reference provider_type: inline::meta-reference config: - service_name: ${env.OTEL_SERVICE_NAME:llama-stack} + service_name: "${env.OTEL_SERVICE_NAME:\u200B}" sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/lmstudio/trace_store.db} tool_runtime: @@ -125,9 +125,9 @@ models: provider_model_id: llama-3.2-3b-instruct model_type: llm - metadata: {} - model_id: llama-3.2-70b-instruct + model_id: llama-3.3-70b-instruct provider_id: lmstudio - provider_model_id: llama-3.2-70b-instruct + provider_model_id: llama-3.3-70b-instruct model_type: llm - metadata: embedding_dimension: 768 From a083465ba4ea4034e31da35142ab983058e2cbd3 Mon Sep 17 00:00:00 2001 From: Matt Clayton Date: Mon, 28 Apr 2025 09:21:23 -0400 Subject: [PATCH 15/15] Add openai completion/chat completion --- .../remote/inference/lmstudio/lmstudio.py | 145 ++++++++++++++++-- 1 file changed, 136 insertions(+), 9 deletions(-) diff --git a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py index 8f7377bc9..59ec68fe3 100644 --- a/llama_stack/providers/remote/inference/lmstudio/lmstudio.py +++ b/llama_stack/providers/remote/inference/lmstudio/lmstudio.py @@ -4,7 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from typing import AsyncIterator, List, Optional, Union +from typing import Any, AsyncIterator, Dict, List, Optional, Union from llama_stack.apis.common.content_types import ( InterleavedContent, @@ -19,6 +19,11 @@ from llama_stack.apis.inference import ( JsonSchemaResponseFormat, LogProbConfig, Message, + OpenAIChatCompletion, + OpenAIChatCompletionChunk, + OpenAICompletion, + OpenAIMessageParam, + OpenAIResponseFormatParam, ResponseFormat, SamplingParams, TextTruncation, @@ -51,17 +56,139 @@ class LMStudioInferenceAdapter(Inference, ModelsProtocolPrivate): def client(self) -> LMStudioClient: return LMStudioClient(url=self.url) - async def batch_chat_completion(self, *args, **kwargs): - raise NotImplementedError("Batch chat completion not supported by LM Studio Provider") + async def batch_completion( + self, + model_id: str, + content_batch: List[InterleavedContent], + sampling_params: Optional[SamplingParams] = None, + response_format: Optional[ResponseFormat] = None, + logprobs: Optional[LogProbConfig] = None, + ): + raise NotImplementedError("Batch completion is not supported by LM Studio Provider") - async def batch_completion(self, *args, **kwargs): - raise NotImplementedError("Batch completion not supported by LM Studio Provider") + async def batch_chat_completion( + self, + model_id: str, + messages_batch: List[List[Message]], + sampling_params: Optional[SamplingParams] = None, + tools: Optional[List[ToolDefinition]] = None, + tool_config: Optional[ToolConfig] = None, + response_format: Optional[ResponseFormat] = None, + logprobs: Optional[LogProbConfig] = None, + ): + raise NotImplementedError("Batch completion is not supported by LM Studio Provider") - async def openai_chat_completion(self, *args, **kwargs): - raise NotImplementedError("OpenAI chat completion not supported by LM Studio Provider") + async def openai_chat_completion( + self, + model: str, + messages: List[OpenAIMessageParam], + frequency_penalty: Optional[float] = None, + function_call: Optional[Union[str, Dict[str, Any]]] = None, + functions: Optional[List[Dict[str, Any]]] = None, + logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + max_completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + parallel_tool_calls: Optional[bool] = None, + presence_penalty: Optional[float] = None, + response_format: Optional[OpenAIResponseFormatParam] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stream: Optional[bool] = None, + stream_options: Optional[Dict[str, Any]] = None, + temperature: Optional[float] = None, + tool_choice: Optional[Union[str, Dict[str, Any]]] = None, + tools: Optional[List[Dict[str, Any]]] = None, + top_logprobs: Optional[int] = None, + top_p: Optional[float] = None, + user: Optional[str] = None, + ) -> Union[OpenAIChatCompletion, AsyncIterator[OpenAIChatCompletionChunk]]: + if self.model_store is None: + raise ValueError("ModelStore is not initialized") + model_obj = await self.model_store.get_model(model) + params = { + k: v + for k, v in { + "model": model_obj.provider_resource_id, + "messages": messages, + "frequency_penalty": frequency_penalty, + "function_call": function_call, + "functions": functions, + "logit_bias": logit_bias, + "logprobs": logprobs, + "max_completion_tokens": max_completion_tokens, + "max_tokens": max_tokens, + "n": n, + "parallel_tool_calls": parallel_tool_calls, + "presence_penalty": presence_penalty, + "response_format": response_format, + "seed": seed, + "stop": stop, + "stream": stream, + "stream_options": stream_options, + "temperature": temperature, + "tool_choice": tool_choice, + "tools": tools, + "top_logprobs": top_logprobs, + "top_p": top_p, + "user": user, + }.items() + if v is not None + } + return await self.openai_client.chat.completions.create(**params) # type: ignore - async def openai_completion(self, *args, **kwargs): - raise NotImplementedError("OpenAI completion not supported by LM Studio Provider") + async def openai_completion( + self, + model: str, + prompt: Union[str, List[str], List[int], List[List[int]]], + best_of: Optional[int] = None, + echo: Optional[bool] = None, + frequency_penalty: Optional[float] = None, + logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + presence_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stream: Optional[bool] = None, + stream_options: Optional[Dict[str, Any]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + user: Optional[str] = None, + guided_choice: Optional[List[str]] = None, + prompt_logprobs: Optional[int] = None, + ) -> OpenAICompletion: + if not isinstance(prompt, str): + raise ValueError("LM Studio does not support non-string prompts for completion") + if self.model_store is None: + raise ValueError("ModelStore is not initialized") + model_obj = await self.model_store.get_model(model) + params = { + k: v + for k, v in { + "model": model_obj.provider_resource_id, + "prompt": prompt, + "best_of": best_of, + "echo": echo, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "logprobs": logprobs, + "max_tokens": max_tokens, + "n": n, + "presence_penalty": presence_penalty, + "seed": seed, + "stop": stop, + "stream": stream, + "stream_options": stream_options, + "temperature": temperature, + "top_p": top_p, + "user": user, + }.items() + if v is not None + } + return await self.openai_client.completions.create(**params) # type: ignore async def initialize(self) -> None: pass