From 19123ca957d95fe19133508a1de53e3b3c86d9c1 Mon Sep 17 00:00:00 2001
From: Nathan Weinberg <31703736+nathan-weinberg@users.noreply.github.com>
Date: Tue, 12 Aug 2025 06:20:39 -0400
Subject: [PATCH 01/45] refactor: standardize InferenceRouter model handling
 (#2965)

---
 llama_stack/apis/common/errors.py             | 10 ++++
 llama_stack/core/routers/inference.py         | 49 ++++++-------------
 llama_stack/core/routing_tables/vector_dbs.py |  4 +-
 .../remote/inference/ollama/ollama.py         |  3 --
 4 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/llama_stack/apis/common/errors.py b/llama_stack/apis/common/errors.py
index 95d6ac18e..6e0fa0b3c 100644
--- a/llama_stack/apis/common/errors.py
+++ b/llama_stack/apis/common/errors.py
@@ -62,3 +62,13 @@ class SessionNotFoundError(ValueError):
     def __init__(self, session_name: str) -> None:
         message = f"Session '{session_name}' not found or access denied."
         super().__init__(message)
+
+
+class ModelTypeError(TypeError):
+    """raised when a model is present but not the correct type"""
+
+    def __init__(self, model_name: str, model_type: str, expected_model_type: str) -> None:
+        message = (
+            f"Model '{model_name}' is of type '{model_type}' rather than the expected type '{expected_model_type}'"
+        )
+        super().__init__(message)
diff --git a/llama_stack/core/routers/inference.py b/llama_stack/core/routers/inference.py
index 79ab7c34f..52581cc9d 100644
--- a/llama_stack/core/routers/inference.py
+++ b/llama_stack/core/routers/inference.py
@@ -18,7 +18,7 @@ from llama_stack.apis.common.content_types import (
     InterleavedContent,
     InterleavedContentItem,
 )
-from llama_stack.apis.common.errors import ModelNotFoundError
+from llama_stack.apis.common.errors import ModelNotFoundError, ModelTypeError
 from llama_stack.apis.inference import (
     BatchChatCompletionResponse,
     BatchCompletionResponse,
@@ -177,6 +177,15 @@ class InferenceRouter(Inference):
             encoded = self.formatter.encode_content(messages)
         return len(encoded.tokens) if encoded and encoded.tokens else 0
 
+    async def _get_model(self, model_id: str, expected_model_type: str) -> Model:
+        """takes a model id and gets model after ensuring that it is accessible and of the correct type"""
+        model = await self.routing_table.get_model(model_id)
+        if model is None:
+            raise ModelNotFoundError(model_id)
+        if model.model_type != expected_model_type:
+            raise ModelTypeError(model_id, model.model_type, expected_model_type)
+        return model
+
     async def chat_completion(
         self,
         model_id: str,
@@ -195,11 +204,7 @@ class InferenceRouter(Inference):
         )
         if sampling_params is None:
             sampling_params = SamplingParams()
-        model = await self.routing_table.get_model(model_id)
-        if model is None:
-            raise ModelNotFoundError(model_id)
-        if model.model_type == ModelType.embedding:
-            raise ValueError(f"Model '{model_id}' is an embedding model and does not support chat completions")
+        model = await self._get_model(model_id, ModelType.llm)
         if tool_config:
             if tool_choice and tool_choice != tool_config.tool_choice:
                 raise ValueError("tool_choice and tool_config.tool_choice must match")
@@ -301,11 +306,7 @@ class InferenceRouter(Inference):
         logger.debug(
             f"InferenceRouter.completion: {model_id=}, {stream=}, {content=}, {sampling_params=}, {response_format=}",
         )
-        model = await self.routing_table.get_model(model_id)
-        if model is None:
-            raise ModelNotFoundError(model_id)
-        if model.model_type == ModelType.embedding:
-            raise ValueError(f"Model '{model_id}' is an embedding model and does not support chat completions")
+        model = await self._get_model(model_id, ModelType.llm)
         provider = await self.routing_table.get_provider_impl(model_id)
         params = dict(
             model_id=model_id,
@@ -355,11 +356,7 @@ class InferenceRouter(Inference):
         task_type: EmbeddingTaskType | None = None,
     ) -> EmbeddingsResponse:
         logger.debug(f"InferenceRouter.embeddings: {model_id}")
-        model = await self.routing_table.get_model(model_id)
-        if model is None:
-            raise ModelNotFoundError(model_id)
-        if model.model_type == ModelType.llm:
-            raise ValueError(f"Model '{model_id}' is an LLM model and does not support embeddings")
+        await self._get_model(model_id, ModelType.embedding)
         provider = await self.routing_table.get_provider_impl(model_id)
         return await provider.embeddings(
             model_id=model_id,
@@ -395,12 +392,7 @@ class InferenceRouter(Inference):
         logger.debug(
             f"InferenceRouter.openai_completion: {model=}, {stream=}, {prompt=}",
         )
-        model_obj = await self.routing_table.get_model(model)
-        if model_obj is None:
-            raise ModelNotFoundError(model)
-        if model_obj.model_type == ModelType.embedding:
-            raise ValueError(f"Model '{model}' is an embedding model and does not support completions")
-
+        model_obj = await self._get_model(model, ModelType.llm)
         params = dict(
             model=model_obj.identifier,
             prompt=prompt,
@@ -476,11 +468,7 @@ class InferenceRouter(Inference):
         logger.debug(
             f"InferenceRouter.openai_chat_completion: {model=}, {stream=}, {messages=}",
         )
-        model_obj = await self.routing_table.get_model(model)
-        if model_obj is None:
-            raise ModelNotFoundError(model)
-        if model_obj.model_type == ModelType.embedding:
-            raise ValueError(f"Model '{model}' is an embedding model and does not support chat completions")
+        model_obj = await self._get_model(model, ModelType.llm)
 
         # Use the OpenAI client for a bit of extra input validation without
         # exposing the OpenAI client itself as part of our API surface
@@ -567,12 +555,7 @@ class InferenceRouter(Inference):
         logger.debug(
             f"InferenceRouter.openai_embeddings: {model=}, input_type={type(input)}, {encoding_format=}, {dimensions=}",
         )
-        model_obj = await self.routing_table.get_model(model)
-        if model_obj is None:
-            raise ModelNotFoundError(model)
-        if model_obj.model_type != ModelType.embedding:
-            raise ValueError(f"Model '{model}' is not an embedding model")
-
+        model_obj = await self._get_model(model, ModelType.embedding)
         params = dict(
             model=model_obj.identifier,
             input=input,
diff --git a/llama_stack/core/routing_tables/vector_dbs.py b/llama_stack/core/routing_tables/vector_dbs.py
index c81a27a3b..e8dc46997 100644
--- a/llama_stack/core/routing_tables/vector_dbs.py
+++ b/llama_stack/core/routing_tables/vector_dbs.py
@@ -8,7 +8,7 @@ from typing import Any
 
 from pydantic import TypeAdapter
 
-from llama_stack.apis.common.errors import ModelNotFoundError, VectorStoreNotFoundError
+from llama_stack.apis.common.errors import ModelNotFoundError, ModelTypeError, VectorStoreNotFoundError
 from llama_stack.apis.models import ModelType
 from llama_stack.apis.resource import ResourceType
 from llama_stack.apis.vector_dbs import ListVectorDBsResponse, VectorDB, VectorDBs
@@ -66,7 +66,7 @@ class VectorDBsRoutingTable(CommonRoutingTableImpl, VectorDBs):
         if model is None:
             raise ModelNotFoundError(embedding_model)
         if model.model_type != ModelType.embedding:
-            raise ValueError(f"Model {embedding_model} is not an embedding model")
+            raise ModelTypeError(embedding_model, model.model_type, ModelType.embedding)
         if "embedding_dimension" not in model.metadata:
             raise ValueError(f"Model {embedding_model} does not have an embedding dimension")
         vector_db_data = {
diff --git a/llama_stack/providers/remote/inference/ollama/ollama.py b/llama_stack/providers/remote/inference/ollama/ollama.py
index 26b4dec76..a93421536 100644
--- a/llama_stack/providers/remote/inference/ollama/ollama.py
+++ b/llama_stack/providers/remote/inference/ollama/ollama.py
@@ -457,9 +457,6 @@ class OllamaInferenceAdapter(
         user: str | None = None,
     ) -> OpenAIEmbeddingsResponse:
         model_obj = await self._get_model(model)
-        if model_obj.model_type != ModelType.embedding:
-            raise ValueError(f"Model {model} is not an embedding model")
-
         if model_obj.provider_resource_id is None:
             raise ValueError(f"Model {model} has no provider_resource_id set")
 

From 4a13ef45e984af8274b06e118af3baadf770a3bc Mon Sep 17 00:00:00 2001
From: Mustafa Elbehery <melbeher@redhat.com>
Date: Tue, 12 Aug 2025 17:32:52 +0200
Subject: [PATCH 02/45] fix: Implement missing `run_moderation` method in
 `PromptGuardSafetyImpl` (#3101)

# What does this PR do?
<!-- Provide a short summary of what this PR does and why. Link to
relevant issues if applicable. -->
This PR addresses an issue where `PromptGuardSafetyImpl` was an
incomplete implementation of an abstract class. The class was missing
the required run_moderation method from its parent interface.


Currently, running `pre-commit` locally fails with the error below.

```
llama_stack/providers/inline/safety/prompt_guard/__init__.py:15: error: Cannot instantiate abstract class "PromptGuardSafetyImpl" with abstract attribute "run_moderation"  [abstract]
Found 1 error in 1 file (checked 410 source files)
```

This PR fixes the issue as follows

- Added the missing run_moderation method to PromptGuardSafetyImpl
- Method raises NotImplementedError with appropriate message indicating
this functionality is not implemented for PromptGuard
- This allows the class to be properly instantiated while clearly
indicating the limitation

<!-- If resolving an issue, uncomment and update the line below -->
<!-- Closes #[issue-number] -->

Signed-off-by: Mustafa Elbehery <melbeher@redhat.com>
---
 .../providers/inline/safety/prompt_guard/prompt_guard.py       | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py b/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py
index e11ec5cf5..801500dee 100644
--- a/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py
+++ b/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py
@@ -64,6 +64,9 @@ class PromptGuardSafetyImpl(Safety, ShieldsProtocolPrivate):
 
         return await self.shield.run(messages)
 
+    async def run_moderation(self, input: str | list[str], model: str):
+        raise NotImplementedError("run_moderation not implemented for PromptGuard")
+
 
 class PromptGuardShield:
     def __init__(

From b70e2f1f09bae55603674e56de46a62608ee588e Mon Sep 17 00:00:00 2001
From: Matthew Farrellee <matt@cs.wisc.edu>
Date: Tue, 12 Aug 2025 10:40:32 -0500
Subject: [PATCH 03/45] fix(dep): update to openai >= 1.99.6 and use new
 Function location (#3087)

# What does this PR do?

closes #3072

## Test Plan

ci
---
 llama_stack/providers/utils/inference/openai_compat.py | 2 +-
 pyproject.toml                                         | 2 +-
 uv.lock                                                | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py
index e6e5ccc8a..9a77c8cc4 100644
--- a/llama_stack/providers/utils/inference/openai_compat.py
+++ b/llama_stack/providers/utils/inference/openai_compat.py
@@ -70,7 +70,7 @@ from openai.types.chat.chat_completion_chunk import (
 from openai.types.chat.chat_completion_content_part_image_param import (
     ImageURL as OpenAIImageURL,
 )
-from openai.types.chat.chat_completion_message_tool_call_param import (
+from openai.types.chat.chat_completion_message_tool_call import (
     Function as OpenAIFunction,
 )
 from pydantic import BaseModel
diff --git a/pyproject.toml b/pyproject.toml
index a77ec5ac9..1b0850631 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,7 +33,7 @@ dependencies = [
     "jsonschema",
     "llama-stack-client>=0.2.17",
     "llama-api-client>=0.1.2",
-    "openai>=1.66",
+    "openai>=1.99.6",
     "prompt-toolkit",
     "python-dotenv",
     "python-jose[cryptography]",
diff --git a/uv.lock b/uv.lock
index c10a7962c..f57b5a161 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1674,7 +1674,7 @@ requires-dist = [
     { name = "llama-api-client", specifier = ">=0.1.2" },
     { name = "llama-stack-client", specifier = ">=0.2.17" },
     { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.17" },
-    { name = "openai", specifier = ">=1.66" },
+    { name = "openai", specifier = ">=1.99.6" },
     { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" },
     { name = "opentelemetry-sdk", specifier = ">=1.30.0" },
     { name = "pandas", marker = "extra == 'ui'" },
@@ -2301,7 +2301,7 @@ wheels = [
 
 [[package]]
 name = "openai"
-version = "1.98.0"
+version = "1.99.6"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "anyio" },
@@ -2313,9 +2313,9 @@ dependencies = [
     { name = "tqdm" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/d8/9d/52eadb15c92802711d6b6cf00df3a6d0d18b588f4c5ba5ff210c6419fc03/openai-1.98.0.tar.gz", hash = "sha256:3ee0fcc50ae95267fd22bd1ad095ba5402098f3df2162592e68109999f685427", size = 496695, upload-time = "2025-07-30T12:48:03.701Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/11/45/38a87bd6949236db5ae3132f41d5861824702b149f86d2627d6900919103/openai-1.99.6.tar.gz", hash = "sha256:f48f4239b938ef187062f3d5199a05b69711d8b600b9a9b6a3853cd271799183", size = 505364, upload-time = "2025-08-09T15:20:54.438Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a8/fe/f64631075b3d63a613c0d8ab761d5941631a470f6fa87eaaee1aa2b4ec0c/openai-1.98.0-py3-none-any.whl", hash = "sha256:b99b794ef92196829120e2df37647722104772d2a74d08305df9ced5f26eae34", size = 767713, upload-time = "2025-07-30T12:48:01.264Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/dd/9aa956485c2856346b3181542fbb0aea4e5b457fa7a523944726746da8da/openai-1.99.6-py3-none-any.whl", hash = "sha256:e40d44b2989588c45ce13819598788b77b8fb80ba2f7ae95ce90d14e46f1bd26", size = 786296, upload-time = "2025-08-09T15:20:51.95Z" },
 ]
 
 [[package]]

From 393f3714b0ea9b71c425ff932510dea4709ea1f7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 12 Aug 2025 08:44:24 -0700
Subject: [PATCH 04/45] chore(python-deps): bump torch from 2.7.1 to 2.8.0
 (#3082)

Bumps [torch](https://github.com/pytorch/pytorch) from 2.7.1 to 2.8.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/pytorch/pytorch/releases">torch's
releases</a>.</em></p>
<blockquote>
<h1>PyTorch 2.8.0 Release Notes</h1>
<ul>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#highlights">Highlights</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#backwards-incompatible-changes">Backwards
Incompatible Changes</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#deprecations">Deprecations</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#new-features">New
Features</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#improvements">Improvements</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#bug-fixes">Bug
fixes</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#performance">Performance</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#documentation">Documentation</a></li>
<li><a
href="https://github.com/pytorch/pytorch/blob/HEAD/#developers">Developers</a></li>
</ul>
<h1>Highlights</h1>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/pytorch/pytorch/commit/ba56102387ef21a3b04b357e5b183d48f0afefc7"><code>ba56102</code></a>
Cherrypick: Add the RunLLM widget to the website (<a
href="https://redirect.github.com/pytorch/pytorch/issues/159592">#159592</a>)</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/c525a02c89217181b5731d8043c7309a84e84066"><code>c525a02</code></a>
[dynamo, docs] cherry pick torch.compile programming model docs into 2.8
(<a
href="https://redirect.github.com/pytorch/pytorch/issues/15">#15</a>...</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/a1cb3cc05d46d198467bebbb6e8fba50a325d4e7"><code>a1cb3cc</code></a>
[Release Only] Remove nvshmem from list of preload libraries (<a
href="https://redirect.github.com/pytorch/pytorch/issues/158925">#158925</a>)</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/c76b2356bc31654de2af0c98cce1bef291f06f89"><code>c76b235</code></a>
Move out super large one off foreach_copy test (<a
href="https://redirect.github.com/pytorch/pytorch/issues/158880">#158880</a>)</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/20a0e225a01d4ebbffd44a6a59acff628359c772"><code>20a0e22</code></a>
Revert &quot;[Dynamo] Allow inlining into AO quantization modules (<a
href="https://redirect.github.com/pytorch/pytorch/issues/152934">#152934</a>)&quot;
(<a
href="https://redirect.github.com/pytorch/pytorch/issues/158">#158</a>...</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/9167ac8c75481e2beb3746aa37b7f48a213c631e"><code>9167ac8</code></a>
[MPS] Switch Cholesky decomp to column wise (<a
href="https://redirect.github.com/pytorch/pytorch/issues/158237">#158237</a>)</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/5534685c62399db8d1e51b47e2dcbc17deaab230"><code>5534685</code></a>
[MPS] Reimplement <code>tri[ul]</code> as Metal shaders (<a
href="https://redirect.github.com/pytorch/pytorch/issues/158867">#158867</a>)</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/d19e08d74b2a27e661bf57a9015014b757e8ea31"><code>d19e08d</code></a>
Cherry pick PR 158746 (<a
href="https://redirect.github.com/pytorch/pytorch/issues/158801">#158801</a>)</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/a6c044ab9aa14f0864c6a572f7c023432511c5ea"><code>a6c044a</code></a>
[cherry-pick] Unify torch.tensor and torch.ops.aten.scalar_tensor
behavior (#...</li>
<li><a
href="https://github.com/pytorch/pytorch/commit/620ebd0646252bbb22524f5c252ec7e9ab977bee"><code>620ebd0</code></a>
[Dynamo] Use proper sources for constructing dataclass defaults (<a
href="https://redirect.github.com/pytorch/pytorch/issues/158689">#158689</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/pytorch/pytorch/compare/v2.7.1...v2.8.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=torch&package-manager=uv&previous-version=2.7.1&new-version=2.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 uv.lock | 75 ++++++++++++++++++++++++++++++---------------------------
 1 file changed, 39 insertions(+), 36 deletions(-)

diff --git a/uv.lock b/uv.lock
index f57b5a161..caafa1197 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1632,10 +1632,10 @@ test = [
     { name = "pypdf" },
     { name = "requests" },
     { name = "sqlalchemy", extra = ["asyncio"] },
-    { name = "torch", version = "2.7.1", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
-    { name = "torch", version = "2.7.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
-    { name = "torchvision", version = "0.22.1", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
-    { name = "torchvision", version = "0.22.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
+    { name = "torch", version = "2.8.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
+    { name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
+    { name = "torchvision", version = "0.23.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
+    { name = "torchvision", version = "0.23.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
     { name = "transformers" },
     { name = "weaviate-client" },
 ]
@@ -4310,7 +4310,7 @@ wheels = [
 
 [[package]]
 name = "torch"
-version = "2.7.1"
+version = "2.8.0"
 source = { registry = "https://download.pytorch.org/whl/cpu" }
 resolution-markers = [
     "python_full_version >= '3.13' and sys_platform == 'darwin'",
@@ -4326,14 +4326,14 @@ dependencies = [
     { name = "typing-extensions", marker = "sys_platform == 'darwin'" },
 ]
 wheels = [
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:7b4f8b2b83bd08f7d399025a9a7b323bdbb53d20566f1e0d584689bb92d82f9a" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:95af97e7b2cecdc89edc0558962a51921bf9c61538597dbec6b7cc48d31e2e13" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:7ecd868a086468e1bcf74b91db425c1c2951a9cfcd0592c4c73377b7e42485ae" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" },
 ]
 
 [[package]]
 name = "torch"
-version = "2.7.1+cpu"
+version = "2.8.0+cpu"
 source = { registry = "https://download.pytorch.org/whl/cpu" }
 resolution-markers = [
     "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')",
@@ -4351,21 +4351,24 @@ dependencies = [
     { name = "typing-extensions", marker = "sys_platform != 'darwin'" },
 ]
 wheels = [
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3bf2db5adf77b433844f080887ade049c4705ddf9fe1a32023ff84ff735aa5ad" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:8f8b3cfc53010a4b4a3c7ecb88c212e9decc4f5eeb6af75c3c803937d2d60947" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:0bc887068772233f532b51a3e8c8cfc682ae62bef74bf4e0c53526c8b9e4138f" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:a2618775f32eb4126c5b2050686da52001a08cffa331637d9cf51c8250931e00" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:eb17646792ac4374ffc87e42369f45d21eff17c790868963b90483ef0b6db4ef" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:84ea1f6a1d15663037d01b121d6e33bb9da3c90af8e069e5072c30f413455a57" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:b66f77f6f67317344ee083aa7ac4751a14395fcb38060d564bf513978d267153" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:56136a2aca6707df3c8811e46ea2d379eaafd18e656e2fd51e8e4d0ca995651b" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:355614185a2aea7155f9c88a20bfd49de5f3063866f3cf9b2f21b6e9e59e31e0" },
-    { url = "https://download.pytorch.org/whl/cpu/torch-2.7.1%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:464bca1bc9452f2ccd676514688896e66b9488f2a0268ecd3ac497cf09c5aac1" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" },
 ]
 
 [[package]]
 name = "torchvision"
-version = "0.22.1"
+version = "0.23.0"
 source = { registry = "https://download.pytorch.org/whl/cpu" }
 resolution-markers = [
     "python_full_version >= '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'",
@@ -4376,21 +4379,21 @@ resolution-markers = [
 dependencies = [
     { name = "numpy", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
     { name = "pillow", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
-    { name = "torch", version = "2.7.1", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
-    { name = "torch", version = "2.7.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
+    { name = "torch", version = "2.8.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
+    { name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
 ]
 wheels = [
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:153f1790e505bd6da123e21eee6e83e2e155df05c0fe7d56347303067d8543c5" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:964414eef19459d55a10e886e2fca50677550e243586d1678f65e3f6f6bac47a" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c3ae3319624c43cc8127020f46c14aa878406781f0899bb6283ae474afeafbf" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:4a614a6a408d2ed74208d0ea6c28a2fbb68290e9a7df206c5fef3f0b6865d307" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:043d9e35ed69c2e586aff6eb9e2887382e7863707115668ac9d140da58f42cba" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:27142bcc8a984227a6dcf560985e83f52b82a7d3f5fe9051af586a2ccc46ef26" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e0e2c04a91403e8dd3af9756c6a024a1d9c0ed9c0d592a8314ded8f4fe30d440" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6dd7c4d329a0e03157803031bc856220c6155ef08c26d4f5bbac938acecf0948" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1c37e325e09a184b730c3ef51424f383ec5745378dc0eca244520aca29722600" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2f7fd6c15f3697e80627b77934f77705f3bc0e98278b989b2655de01f6903e1d" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:2df618e1143805a7673aaf82cb5720dd9112d4e771983156aaf2ffff692eebf9" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2a3299d2b1d5a7aed2d3b6ffb69c672ca8830671967eb1cee1497bacd82fe47b" },
 ]
 
 [[package]]
 name = "torchvision"
-version = "0.22.1+cpu"
+version = "0.23.0+cpu"
 source = { registry = "https://download.pytorch.org/whl/cpu" }
 resolution-markers = [
     "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')",
@@ -4399,15 +4402,15 @@ resolution-markers = [
 dependencies = [
     { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
     { name = "pillow", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
-    { name = "torch", version = "2.7.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
+    { name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
 ]
 wheels = [
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b5fa7044bd82c6358e8229351c98070cf3a7bf4a6e89ea46352ae6c65745ef94" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:433cb4dbced7291f17064cea08ac1e5aebd02ec190e1c207d117ad62a8961f2b" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a93c21f18c33a819616b3dda7655aa4de40b219682c654175b6bbeb65ecc2e5f" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:34c914ad4728b81848ac802c5fc5eeb8de8ff4058cc59c1463a74ce4f4fbf0d8" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ab7ae82529887c704c1b5d1d5198f65dc777d04fc3858b374503a6deedb82b19" },
-    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.22.1%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:b2d1c4bdbfd8e6c779dc810a6171b56224f1332fc46986810d4081bed1633804" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ae459d4509d3b837b978dc6c66106601f916b6d2cda75c137e3f5f48324ce1da" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:a651ccc540cf4c87eb988730c59c2220c52b57adc276f044e7efb9830fa65a1d" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:dea90a67d60a5366b0358a0b8d6bf267805278697d6fd950cf0e31139e56d1be" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:82928788025170c62e7df1120dcdc0cd175bfc31c08374613ce6d1a040bc0cda" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:474d77adbbbed5166db3e5636b4b4ae3399c66ef5bfa12536e254b32259c90c0" },
+    { url = "https://download.pytorch.org/whl/cpu/torchvision-0.23.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:8d6a47e23d7896f0ef9aa7ea7179eb6324e82438aa66d19884c2020d0646b104" },
 ]
 
 [[package]]

From 88c4fdc5d7fecd0468815f9eda25d72f722745a6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 12 Aug 2025 08:44:39 -0700
Subject: [PATCH 05/45] chore(python-deps): bump chromadb from 1.0.15 to 1.0.16
 (#3083)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [chromadb](https://github.com/chroma-core/chroma) from 1.0.15 to
1.0.16.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/chroma-core/chroma/releases">chromadb's
releases</a>.</em></p>
<blockquote>
<h2>1.0.16</h2>
<p>Version: <code>1.0.16</code>
Git ref: <code>refs/tags/1.0.16</code>
Build Date: <code>2025-08-08T00:26</code>
PIP Package: <code>chroma-1.0.16.tar.gz</code>
Github Container Registry Image: <code>:1.0.16</code>
DockerHub Image: <code>:1.0.16</code></p>
<h2>What's Changed</h2>
<ul>
<li>[ENH]: add cache mount &amp; tolerations to garbage collector
template in Helm chart by <a
href="https://github.com/codetheweb"><code>@​codetheweb</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5016">chroma-core/chroma#5016</a></li>
<li>[DOC] Fix docs typo by <a
href="https://github.com/itaismith"><code>@​itaismith</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5018">chroma-core/chroma#5018</a></li>
<li>[CLN] Change GenericQuotaError from 429 to 422 by <a
href="https://github.com/drewkim"><code>@​drewkim</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5022">chroma-core/chroma#5022</a></li>
<li>[CHORE] Fix type error in batch_utils by <a
href="https://github.com/jairad26"><code>@​jairad26</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5024">chroma-core/chroma#5024</a></li>
<li>[ENH] Add block-level metrics by <a
href="https://github.com/tanujnay112"><code>@​tanujnay112</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/4801">chroma-core/chroma#4801</a></li>
<li>[ENH]: return error on /add if embeddings are not provided by <a
href="https://github.com/codetheweb"><code>@​codetheweb</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5033">chroma-core/chroma#5033</a></li>
<li>[DOC] Docs Polish 07/2025 by <a
href="https://github.com/itaismith"><code>@​itaismith</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5032">chroma-core/chroma#5032</a></li>
<li>[DOC] Flatten public txt files by <a
href="https://github.com/itaismith"><code>@​itaismith</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5040">chroma-core/chroma#5040</a></li>
<li>[ENH]: require embeddings &amp; require min embedding dimension on
/add by <a
href="https://github.com/codetheweb"><code>@​codetheweb</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5037">chroma-core/chroma#5037</a></li>
<li>[ENH] - Adds in dark mode support for hero image by <a
href="https://github.com/tjkrusinskichroma"><code>@​tjkrusinskichroma</code></a>
in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5042">chroma-core/chroma#5042</a></li>
<li>[BLD] Use 8core runners for all our windows jobs by <a
href="https://github.com/eculver"><code>@​eculver</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5027">chroma-core/chroma#5027</a></li>
<li>[TST] More benchmark queries for regex by <a
href="https://github.com/Sicheng-Pan"><code>@​Sicheng-Pan</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/4910">chroma-core/chroma#4910</a></li>
<li>[BUG]: refactor otel/tracing initialization in the frontend to be
independent of hosted entry point by <a
href="https://github.com/c-gamble"><code>@​c-gamble</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5028">chroma-core/chroma#5028</a></li>
<li>[BUG] js client: handle 422 billing errors as QuotaExceeded instead
of ChromaConnectionError by <a
href="https://github.com/philipithomas"><code>@​philipithomas</code></a>
in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5049">chroma-core/chroma#5049</a></li>
<li>[BUG] RLS should use 32MB GRPC payload size limit by <a
href="https://github.com/Sicheng-Pan"><code>@​Sicheng-Pan</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5044">chroma-core/chroma#5044</a></li>
<li>[BUG] Sync protoc arch and version in dockerfile by <a
href="https://github.com/Sicheng-Pan"><code>@​Sicheng-Pan</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5045">chroma-core/chroma#5045</a></li>
<li>[BLD] Fix windows runner label by <a
href="https://github.com/eculver"><code>@​eculver</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5052">chroma-core/chroma#5052</a></li>
<li>[PERF]: Prefetch segments in get and query by <a
href="https://github.com/sanketkedia"><code>@​sanketkedia</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5053">chroma-core/chroma#5053</a></li>
<li>[PERF]: Parallelize fetching blocks for brute force regex by <a
href="https://github.com/sanketkedia"><code>@​sanketkedia</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5051">chroma-core/chroma#5051</a></li>
<li>[RELEASE] JS 3.0.7 by <a
href="https://github.com/itaismith"><code>@​itaismith</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5059">chroma-core/chroma#5059</a></li>
<li>[ENH] Add a delete_many call to the storage API. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5020">chroma-core/chroma#5020</a></li>
<li>[ENH] Consume delete_many from the wal3 garbage collector. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5021">chroma-core/chroma#5021</a></li>
<li>[ENH]: limit number of concurrent get_all_block_ids() when using
buffer_unordered() by <a
href="https://github.com/codetheweb"><code>@​codetheweb</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5062">chroma-core/chroma#5062</a></li>
<li>[ENH]: use new <code>delete_many()</code> storage method in
DeleteUnusedFiles operator by <a
href="https://github.com/codetheweb"><code>@​codetheweb</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5061">chroma-core/chroma#5061</a></li>
<li>[BUG]: Disable aws stalled stream protection by <a
href="https://github.com/tanujnay112"><code>@​tanujnay112</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5063">chroma-core/chroma#5063</a></li>
<li>[DOC] Update manage collections docs with correct delete collection
info by <a
href="https://github.com/jairad26"><code>@​jairad26</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5066">chroma-core/chroma#5066</a></li>
<li>[BUG] Improve wal3 robustness with better shutdown handling and
error recovery by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5046">chroma-core/chroma#5046</a></li>
<li>[ENH] Do not do any mutations of the manifest from within GC. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5050">chroma-core/chroma#5050</a></li>
<li>[CHORE]: enable change notifier otel/tracing by <a
href="https://github.com/c-gamble"><code>@​c-gamble</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5073">chroma-core/chroma#5073</a></li>
<li>[CHORE] Add pprof server to query service by <a
href="https://github.com/eculver"><code>@​eculver</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5072">chroma-core/chroma#5072</a></li>
<li>[ENH]: Dedup inserts to the same key in foyer by <a
href="https://github.com/sanketkedia"><code>@​sanketkedia</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5074">chroma-core/chroma#5074</a></li>
<li>[ENH] &quot;Failed to fetch: status: NotFound&quot; be gone. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5064">chroma-core/chroma#5064</a></li>
<li>[CLN] Remove the the top most spammy log lines from rls/wal3. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5071">chroma-core/chroma#5071</a></li>
<li>[DOC] Fix badge in readme by <a
href="https://github.com/kylediaz"><code>@​kylediaz</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5025">chroma-core/chroma#5025</a></li>
<li>[ENH] A tool for patching logs that were deleted before a new
manifest was installed. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5083">chroma-core/chroma#5083</a></li>
<li>[BUG] Add billing errors to JS client by <a
href="https://github.com/itaismith"><code>@​itaismith</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5084">chroma-core/chroma#5084</a></li>
<li>[CHORE]: Add s3 get metrics and pod name to tracing spans by <a
href="https://github.com/tanujnay112"><code>@​tanujnay112</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5086">chroma-core/chroma#5086</a></li>
<li>[RELEASE] JS 3.0.8 by <a
href="https://github.com/itaismith"><code>@​itaismith</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5087">chroma-core/chroma#5087</a></li>
<li>[ENH] A tool to purge the cache. by <a
href="https://github.com/rescrv"><code>@​rescrv</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5085">chroma-core/chroma#5085</a></li>
<li>[DOC] Update PR template for migration and observability by <a
href="https://github.com/HammadB"><code>@​HammadB</code></a> in <a
href="https://redirect.github.com/chroma-core/chroma/pull/5089">chroma-core/chroma#5089</a></li>
<li>[CHORE]: Fix s3 get metric name by <a
href="https://github.com/tanujnay112"><code>@​tanujnay112</code></a> in
<a
href="https://redirect.github.com/chroma-core/chroma/pull/5091">chroma-core/chroma#5091</a></li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/chroma-core/chroma/commit/dff3a786db730da7164b534106ca151da8809060"><code>dff3a78</code></a>
[RELEASE] CLI 1.1.5, Python 1.0.16, JS 3.0.11 (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5227">#5227</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/f60f932b8df63274ff4fdd6a4139fef494aa2646"><code>f60f932</code></a>
[ENH]: Increase nprobe for smaller collections (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5226">#5226</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/f593a43b5d944cea6bd0f0ab6575bc39c3df9f90"><code>f593a43</code></a>
[ENH] Add <code>InsertRecordSet</code> to JS client (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5225">#5225</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/76a14c226aa2740107a186de33ad9ec6537b060c"><code>76a14c2</code></a>
[DOC] Made light/dark mode for Chroma logo (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5215">#5215</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/d80817ede4b650ef2a125ba4171812d8c5b815bb"><code>d80817e</code></a>
[ENH]: Add more tracing in the filter path (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5219">#5219</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/73abfdc51ad4c22486e0bed5e63db1d22a52487d"><code>73abfdc</code></a>
[ENH] Handle when the garbage doesn't overlap the manifest. (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5207">#5207</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/fa392226ba537257fbf7ad85ebc1b57bebfa035a"><code>fa39222</code></a>
[BUG] Revert accidentally commited code (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5205">#5205</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/815c3ac561b207bad30bdf374ad9291e1bf5b0d1"><code>815c3ac</code></a>
[ENH]: Fix CI flake with adaptive nsearch (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5203">#5203</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/ea66d6929ccc0076ec5649c198d4e888137eedb4"><code>ea66d69</code></a>
[BUG] Switch to rust-tls (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5204">#5204</a>)</li>
<li><a
href="https://github.com/chroma-core/chroma/commit/04aeb2213989878260efd7d53063c7e664c202ca"><code>04aeb22</code></a>
[ENH]: Calculate cache weight of block size instead of hardcoding (<a
href="https://redirect.github.com/chroma-core/chroma/issues/5201">#5201</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/chroma-core/chroma/compare/1.0.15...1.0.16">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=chromadb&package-manager=uv&previous-version=1.0.15&new-version=1.0.16)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 uv.lock | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/uv.lock b/uv.lock
index caafa1197..9f4ba4adb 100644
--- a/uv.lock
+++ b/uv.lock
@@ -476,7 +476,7 @@ wheels = [
 
 [[package]]
 name = "chromadb"
-version = "1.0.15"
+version = "1.0.16"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "bcrypt" },
@@ -507,13 +507,13 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "uvicorn", extra = ["standard"] },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ad/e2/0653b2e539db5512d2200c759f1bc7f9ef5609fe47f3c7d24b82f62dc00f/chromadb-1.0.15.tar.gz", hash = "sha256:3e910da3f5414e2204f89c7beca1650847f2bf3bd71f11a2e40aad1eb31050aa", size = 1218840, upload-time = "2025-07-02T17:07:09.875Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/15/2a/5b7e793d2a27c425e9f1813e9cb965b70e9bda08b69ee15a10e07dc3e59a/chromadb-1.0.16.tar.gz", hash = "sha256:3c864b5beb5e131bdc1f83c0b63a01ec481c6ee52028f088563ffba8478478e1", size = 1241545, upload-time = "2025-08-08T00:25:41.414Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/85/5a/866c6f0c2160cbc8dca0cf77b2fb391dcf435b32a58743da1bc1a08dc442/chromadb-1.0.15-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:51791553014297798b53df4e043e9c30f4e8bd157647971a6bb02b04bfa65f82", size = 18838820, upload-time = "2025-07-02T17:07:07.632Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/18/ff9b58ab5d334f5ecff7fdbacd6761bac467176708fa4d2500ae7c048af0/chromadb-1.0.15-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:48015803c0631c3a817befc276436dc084bb628c37fd4214047212afb2056291", size = 18057131, upload-time = "2025-07-02T17:07:05.15Z" },
-    { url = "https://files.pythonhosted.org/packages/31/49/74e34cc5aeeb25aff2c0ede6790b3671e14c1b91574dd8f98d266a4c5aad/chromadb-1.0.15-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b73cd6fb32fcdd91c577cca16ea6112b691d72b441bb3f2140426d1e79e453a", size = 18595284, upload-time = "2025-07-02T17:06:59.102Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/33/190df917a057067e37f8b48d082d769bed8b3c0c507edefc7b6c6bb577d0/chromadb-1.0.15-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:479f1b401af9e7c20f50642ffb3376abbfd78e2b5b170429f7c79eff52e367db", size = 19526626, upload-time = "2025-07-02T17:07:02.163Z" },
-    { url = "https://files.pythonhosted.org/packages/a1/30/6890da607358993f87a01e80bcce916b4d91515ce865f07dc06845cb472f/chromadb-1.0.15-cp39-abi3-win_amd64.whl", hash = "sha256:e0cb3b93fdc42b1786f151d413ef36299f30f783a30ce08bf0bfb12e552b4190", size = 19520490, upload-time = "2025-07-02T17:07:11.559Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/9d/bffcc814272c9b7982551803b2d45b77f39eeea1b9e965c00c05ee81c649/chromadb-1.0.16-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:144163ce7ca4f4448684d5d0c13ebb37c4d68490ecb60967a95d05cea30e0d2d", size = 18942157, upload-time = "2025-08-08T00:25:38.459Z" },
+    { url = "https://files.pythonhosted.org/packages/58/4e/de0086f3cbcfd667d75d112bb546386803ab5335599bf7099272a675e98b/chromadb-1.0.16-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:4ebcc5894e6fbb6b576452bbf4659746bfe58d9daf99a18363364e9497434bd2", size = 18147831, upload-time = "2025-08-08T00:25:35.546Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/7f/a8aff4ce96281bcb9731d10b2554f41963dd0b47acb4f90a78b2b7c4f199/chromadb-1.0.16-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:937051fc3aae94f7c171503d8f1f7662820aacc75acf45f28d3656c75c5ff1f8", size = 18682195, upload-time = "2025-08-08T00:25:29.654Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/9c/2a97d0257176aae472dff6f1ef1b7050449f384e420120e0f31d2d8f532f/chromadb-1.0.16-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0f5c5ad0c59154a9cab1506b857bab8487b588352e668cf1222c54bb9d52daa", size = 19635695, upload-time = "2025-08-08T00:25:32.68Z" },
+    { url = "https://files.pythonhosted.org/packages/96/8a/f7e810f3cbdc9186ba4e649dc32711b7ab2c23aba37cf61175f731d22293/chromadb-1.0.16-cp39-abi3-win_amd64.whl", hash = "sha256:2528c01bd8b3facca9d0e1ffac866767c386b94604df484fc792ee891c86e09a", size = 19641144, upload-time = "2025-08-08T00:25:43.446Z" },
 ]
 
 [[package]]

From 6812aa1e1e6aad77706f1e063fd4ed6603cf9871 Mon Sep 17 00:00:00 2001
From: Nathan Weinberg <31703736+nathan-weinberg@users.noreply.github.com>
Date: Tue, 12 Aug 2025 11:52:57 -0400
Subject: [PATCH 06/45] chore: bump min python version in docs and tests
 (#3103)

# What does this PR do?
the minimum python version for the project was bumped to 3.12 a couple
months ago, but there remains some artifacts in the repo suggesting we
support >=3.10

Signed-off-by: Nathan Weinberg <nweinber@redhat.com>
---
 docs/source/apis/external.md                               | 4 ++--
 docs/source/providers/external/external-providers-guide.md | 2 +-
 tests/external/llama-stack-api-weather/pyproject.toml      | 2 +-
 tests/external/llama-stack-provider-kaze/pyproject.toml    | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/source/apis/external.md b/docs/source/apis/external.md
index cc13deb9b..5831990b0 100644
--- a/docs/source/apis/external.md
+++ b/docs/source/apis/external.md
@@ -111,7 +111,7 @@ name = "llama-stack-api-weather"
 version = "0.1.0"
 description = "Weather API for Llama Stack"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = ["llama-stack", "pydantic"]
 
 [build-system]
@@ -231,7 +231,7 @@ name = "llama-stack-provider-kaze"
 version = "0.1.0"
 description = "Kaze weather provider for Llama Stack"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = ["llama-stack", "pydantic", "aiohttp"]
 
 [build-system]
diff --git a/docs/source/providers/external/external-providers-guide.md b/docs/source/providers/external/external-providers-guide.md
index 2479d406f..e2d4ebea9 100644
--- a/docs/source/providers/external/external-providers-guide.md
+++ b/docs/source/providers/external/external-providers-guide.md
@@ -226,7 +226,7 @@ uv init
 name = "llama-stack-provider-ollama"
 version = "0.1.0"
 description = "Ollama provider for Llama Stack"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = ["llama-stack", "pydantic", "ollama", "aiohttp"]
 ```
 
diff --git a/tests/external/llama-stack-api-weather/pyproject.toml b/tests/external/llama-stack-api-weather/pyproject.toml
index 566e1e9aa..ac2d8d632 100644
--- a/tests/external/llama-stack-api-weather/pyproject.toml
+++ b/tests/external/llama-stack-api-weather/pyproject.toml
@@ -3,7 +3,7 @@ name = "llama-stack-api-weather"
 version = "0.1.0"
 description = "Weather API for Llama Stack"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = ["llama-stack", "pydantic"]
 
 [build-system]
diff --git a/tests/external/llama-stack-provider-kaze/pyproject.toml b/tests/external/llama-stack-provider-kaze/pyproject.toml
index 7bbf1f843..e2438a18a 100644
--- a/tests/external/llama-stack-provider-kaze/pyproject.toml
+++ b/tests/external/llama-stack-provider-kaze/pyproject.toml
@@ -3,7 +3,7 @@ name = "llama-stack-provider-kaze"
 version = "0.1.0"
 description = "Kaze weather provider for Llama Stack"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = ["llama-stack", "pydantic", "aiohttp"]
 
 [build-system]

From 4fec49dfdb4ecb9a11411ae8a672f63b4c7cca58 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Tue, 12 Aug 2025 10:24:01 -0700
Subject: [PATCH 07/45] feat(responses): add include parameter (#3115)

Well our Responses tests use it so we better include it in the API, no?

I discovered it because I want to make sure `llama-stack-client` can be
used always instead of `openai-python` as the client (we do want to be
_truly_ compatible.)
---
 docs/_static/llama-stack-spec.html                  |  7 +++++++
 docs/_static/llama-stack-spec.yaml                  |  6 ++++++
 llama_stack/apis/agents/agents.py                   |  2 ++
 .../inline/agents/meta_reference/agents.py          | 13 ++++++++++++-
 .../agents/meta_reference/openai_responses.py       |  1 +
 5 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index d480ff592..a16d3fce5 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -8515,6 +8515,13 @@
                             "$ref": "#/components/schemas/OpenAIResponseInputTool"
                         }
                     },
+                    "include": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        },
+                        "description": "(Optional) Additional fields to include in the response."
+                    },
                     "max_infer_iters": {
                         "type": "integer"
                     }
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index 9c0fba554..d5ad66d5e 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -6188,6 +6188,12 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/OpenAIResponseInputTool'
+        include:
+          type: array
+          items:
+            type: string
+          description: >-
+            (Optional) Additional fields to include in the response.
         max_infer_iters:
           type: integer
       additionalProperties: false
diff --git a/llama_stack/apis/agents/agents.py b/llama_stack/apis/agents/agents.py
index e816da766..7dd3e9289 100644
--- a/llama_stack/apis/agents/agents.py
+++ b/llama_stack/apis/agents/agents.py
@@ -706,6 +706,7 @@ class Agents(Protocol):
         temperature: float | None = None,
         text: OpenAIResponseText | None = None,
         tools: list[OpenAIResponseInputTool] | None = None,
+        include: list[str] | None = None,
         max_infer_iters: int | None = 10,  # this is an extension to the OpenAI API
     ) -> OpenAIResponseObject | AsyncIterator[OpenAIResponseObjectStream]:
         """Create a new OpenAI response.
@@ -713,6 +714,7 @@ class Agents(Protocol):
         :param input: Input message(s) to create the response.
         :param model: The underlying LLM used for completions.
         :param previous_response_id: (Optional) if specified, the new response will be a continuation of the previous response. This can be used to easily fork-off new responses from existing responses.
+        :param include: (Optional) Additional fields to include in the response.
         :returns: An OpenAIResponseObject.
         """
         ...
diff --git a/llama_stack/providers/inline/agents/meta_reference/agents.py b/llama_stack/providers/inline/agents/meta_reference/agents.py
index 15695ec48..0f12a0865 100644
--- a/llama_stack/providers/inline/agents/meta_reference/agents.py
+++ b/llama_stack/providers/inline/agents/meta_reference/agents.py
@@ -327,10 +327,21 @@ class MetaReferenceAgentsImpl(Agents):
         temperature: float | None = None,
         text: OpenAIResponseText | None = None,
         tools: list[OpenAIResponseInputTool] | None = None,
+        include: list[str] | None = None,
         max_infer_iters: int | None = 10,
     ) -> OpenAIResponseObject:
         return await self.openai_responses_impl.create_openai_response(
-            input, model, instructions, previous_response_id, store, stream, temperature, text, tools, max_infer_iters
+            input,
+            model,
+            instructions,
+            previous_response_id,
+            store,
+            stream,
+            temperature,
+            text,
+            tools,
+            include,
+            max_infer_iters,
         )
 
     async def list_openai_responses(
diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
index 7eb2b3897..db70bc046 100644
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
@@ -333,6 +333,7 @@ class OpenAIResponsesImpl:
         temperature: float | None = None,
         text: OpenAIResponseText | None = None,
         tools: list[OpenAIResponseInputTool] | None = None,
+        include: list[str] | None = None,
         max_infer_iters: int | None = 10,
     ):
         stream = bool(stream)

From 1721aafc1fc13297d7db6c3bcb0c65344c6a7cd0 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Tue, 12 Aug 2025 10:39:09 -0700
Subject: [PATCH 08/45] feat(responses): type file results properly (#3117)

Another thing our tests implicitly depended on.
---
 docs/_static/llama-stack-spec.html            | 74 +++++++++++++------
 docs/_static/llama-stack-spec.yaml            | 46 ++++++++++--
 llama_stack/apis/agents/openai_responses.py   | 19 ++++-
 .../agents/meta_reference/openai_responses.py | 14 ++--
 4 files changed, 117 insertions(+), 36 deletions(-)

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index a16d3fce5..e2c53d4b0 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -8293,28 +8293,60 @@
                         "type": "array",
                         "items": {
                             "type": "object",
-                            "additionalProperties": {
-                                "oneOf": [
-                                    {
-                                        "type": "null"
+                            "properties": {
+                                "attributes": {
+                                    "type": "object",
+                                    "additionalProperties": {
+                                        "oneOf": [
+                                            {
+                                                "type": "null"
+                                            },
+                                            {
+                                                "type": "boolean"
+                                            },
+                                            {
+                                                "type": "number"
+                                            },
+                                            {
+                                                "type": "string"
+                                            },
+                                            {
+                                                "type": "array"
+                                            },
+                                            {
+                                                "type": "object"
+                                            }
+                                        ]
                                     },
-                                    {
-                                        "type": "boolean"
-                                    },
-                                    {
-                                        "type": "number"
-                                    },
-                                    {
-                                        "type": "string"
-                                    },
-                                    {
-                                        "type": "array"
-                                    },
-                                    {
-                                        "type": "object"
-                                    }
-                                ]
-                            }
+                                    "description": "(Optional) Key-value attributes associated with the file"
+                                },
+                                "file_id": {
+                                    "type": "string",
+                                    "description": "Unique identifier of the file containing the result"
+                                },
+                                "filename": {
+                                    "type": "string",
+                                    "description": "Name of the file containing the result"
+                                },
+                                "score": {
+                                    "type": "number",
+                                    "description": "Relevance score for this search result (between 0 and 1)"
+                                },
+                                "text": {
+                                    "type": "string",
+                                    "description": "Text content of the search result"
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "attributes",
+                                "file_id",
+                                "filename",
+                                "score",
+                                "text"
+                            ],
+                            "title": "OpenAIResponseOutputMessageFileSearchToolCallResults",
+                            "description": "Search results returned by the file search operation."
                         },
                         "description": "(Optional) Search results returned by the file search operation"
                     }
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index d5ad66d5e..85cec3a78 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -6021,14 +6021,44 @@ components:
           type: array
           items:
             type: object
-            additionalProperties:
-              oneOf:
-                - type: 'null'
-                - type: boolean
-                - type: number
-                - type: string
-                - type: array
-                - type: object
+            properties:
+              attributes:
+                type: object
+                additionalProperties:
+                  oneOf:
+                    - type: 'null'
+                    - type: boolean
+                    - type: number
+                    - type: string
+                    - type: array
+                    - type: object
+                description: >-
+                  (Optional) Key-value attributes associated with the file
+              file_id:
+                type: string
+                description: >-
+                  Unique identifier of the file containing the result
+              filename:
+                type: string
+                description: Name of the file containing the result
+              score:
+                type: number
+                description: >-
+                  Relevance score for this search result (between 0 and 1)
+              text:
+                type: string
+                description: Text content of the search result
+            additionalProperties: false
+            required:
+              - attributes
+              - file_id
+              - filename
+              - score
+              - text
+            title: >-
+              OpenAIResponseOutputMessageFileSearchToolCallResults
+            description: >-
+              Search results returned by the file search operation.
           description: >-
             (Optional) Search results returned by the file search operation
       additionalProperties: false
diff --git a/llama_stack/apis/agents/openai_responses.py b/llama_stack/apis/agents/openai_responses.py
index 10cadf38f..8574104dc 100644
--- a/llama_stack/apis/agents/openai_responses.py
+++ b/llama_stack/apis/agents/openai_responses.py
@@ -170,6 +170,23 @@ class OpenAIResponseOutputMessageWebSearchToolCall(BaseModel):
     type: Literal["web_search_call"] = "web_search_call"
 
 
+class OpenAIResponseOutputMessageFileSearchToolCallResults(BaseModel):
+    """Search results returned by the file search operation.
+
+    :param attributes: (Optional) Key-value attributes associated with the file
+    :param file_id: Unique identifier of the file containing the result
+    :param filename: Name of the file containing the result
+    :param score: Relevance score for this search result (between 0 and 1)
+    :param text: Text content of the search result
+    """
+
+    attributes: dict[str, Any]
+    file_id: str
+    filename: str
+    score: float
+    text: str
+
+
 @json_schema_type
 class OpenAIResponseOutputMessageFileSearchToolCall(BaseModel):
     """File search tool call output message for OpenAI responses.
@@ -185,7 +202,7 @@ class OpenAIResponseOutputMessageFileSearchToolCall(BaseModel):
     queries: list[str]
     status: str
     type: Literal["file_search_call"] = "file_search_call"
-    results: list[dict[str, Any]] | None = None
+    results: list[OpenAIResponseOutputMessageFileSearchToolCallResults] | None = None
 
 
 @json_schema_type
diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
index db70bc046..b98ca114f 100644
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
@@ -38,6 +38,7 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseOutputMessageContent,
     OpenAIResponseOutputMessageContentOutputText,
     OpenAIResponseOutputMessageFileSearchToolCall,
+    OpenAIResponseOutputMessageFileSearchToolCallResults,
     OpenAIResponseOutputMessageFunctionToolCall,
     OpenAIResponseOutputMessageMCPListTools,
     OpenAIResponseOutputMessageWebSearchToolCall,
@@ -827,12 +828,13 @@ class OpenAIResponsesImpl:
                         text = result.metadata["chunks"][i] if "chunks" in result.metadata else None
                         score = result.metadata["scores"][i] if "scores" in result.metadata else None
                         message.results.append(
-                            {
-                                "file_id": doc_id,
-                                "filename": doc_id,
-                                "text": text,
-                                "score": score,
-                            }
+                            OpenAIResponseOutputMessageFileSearchToolCallResults(
+                                file_id=doc_id,
+                                filename=doc_id,
+                                text=text,
+                                score=score,
+                                attributes={},
+                            )
                         )
                 if error_exc or (result.error_code and result.error_code > 0) or result.error_message:
                     message.status = "failed"

From 3d901178914fefcedeccb8467aaed014fa8b637a Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Tue, 12 Aug 2025 16:15:53 -0700
Subject: [PATCH 09/45] chore(tests): fix responses and vector_io tests (#3119)

Some fixes to MCP tests. And a bunch of fixes for Vector providers.

I also enabled a bunch of Vector IO tests to be used with
`LlamaStackLibraryClient`

## Test Plan

Run Responses tests with llama stack library client:
```
pytest -s -v tests/integration/non_ci/responses/ --stack-config=server:starter \
   --text-model openai/gpt-4o \
  --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
  -k "client_with_models"
```

Do the same with `-k openai_client`

The rest should be taken care of by CI.
---
 .github/actions/setup-runner/action.yml       |  2 +-
 .github/workflows/integration-tests.yml       |  3 +-
 .../workflows/integration-vector-io-tests.yml |  4 +-
 llama_stack/core/build.py                     |  2 +-
 llama_stack/core/routers/inference.py         |  3 +-
 llama_stack/core/routing_tables/models.py     |  2 +
 llama_stack/log.py                            |  1 +
 .../models/llama/llama3/chat_format.py        |  1 +
 .../agents/meta_reference/openai_responses.py |  4 ++
 .../providers/inline/vector_io/faiss/faiss.py | 19 ++++---
 .../inline/vector_io/sqlite_vec/sqlite_vec.py | 25 +++++----
 llama_stack/providers/registry/vector_io.py   |  4 ++
 .../remote/inference/fireworks/fireworks.py   |  3 +
 .../remote/vector_io/chroma/chroma.py         | 17 ++++--
 .../remote/vector_io/milvus/milvus.py         | 16 +++---
 .../remote/vector_io/pgvector/pgvector.py     | 12 ++--
 .../remote/vector_io/qdrant/qdrant.py         | 18 +++---
 .../remote/vector_io/weaviate/weaviate.py     | 11 ++--
 .../utils/memory/openai_vector_store_mixin.py | 32 ++++++++---
 .../providers/utils/memory/vector_store.py    | 15 ++++-
 tests/common/mcp.py                           | 12 +---
 tests/integration/fixtures/common.py          |  2 +-
 .../fixtures/test_cases/responses.yaml        |  6 +-
 .../non_ci/responses/test_responses.py        | 17 ++++--
 .../vector_io/test_openai_vector_stores.py    | 56 ++++++++-----------
 25 files changed, 175 insertions(+), 112 deletions(-)

diff --git a/.github/actions/setup-runner/action.yml b/.github/actions/setup-runner/action.yml
index 0be999fe2..1ca02bbff 100644
--- a/.github/actions/setup-runner/action.yml
+++ b/.github/actions/setup-runner/action.yml
@@ -28,7 +28,7 @@ runs:
         # Install llama-stack-client-python based on the client-version input
         if [ "${{ inputs.client-version }}" = "latest" ]; then
           echo "Installing latest llama-stack-client-python from main branch"
-          uv pip install git+https://github.com/meta-llama/llama-stack-client-python.git@main
+          uv pip install git+https://github.com/llamastack/llama-stack-client-python.git@main
         elif [ "${{ inputs.client-version }}" = "published" ]; then
           echo "Installing published llama-stack-client-python from PyPI"
           uv pip install llama-stack-client
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index a38d4971a..9ef49fba3 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -52,7 +52,8 @@ jobs:
         run: |
           # Get test directories dynamically, excluding non-test directories
           # NOTE: we are excluding post_training since the tests take too long
-          TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
+          TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d |
+            sed 's|tests/integration/||' |
             grep -Ev "^(__pycache__|fixtures|test_cases|recordings|non_ci|post_training)$" |
             sort | jq -R -s -c 'split("\n")[:-1]')
           echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/integration-vector-io-tests.yml b/.github/workflows/integration-vector-io-tests.yml
index aa239572b..f4d28e407 100644
--- a/.github/workflows/integration-vector-io-tests.yml
+++ b/.github/workflows/integration-vector-io-tests.yml
@@ -164,9 +164,9 @@ jobs:
           ENABLE_WEAVIATE: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'true' || '' }}
           WEAVIATE_CLUSTER_URL: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'localhost:8080' || '' }}
         run: |
-          uv run pytest -sv --stack-config="inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \
+          uv run pytest -sv --stack-config="files=inline::localfs,inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \
             tests/integration/vector_io \
-            --embedding-model sentence-transformers/all-MiniLM-L6-v2
+            --embedding-model inline::sentence-transformers/all-MiniLM-L6-v2
 
       - name: Check Storage and Memory Available After Tests
         if: ${{ always() }}
diff --git a/llama_stack/core/build.py b/llama_stack/core/build.py
index b3e35ecef..4b20588fd 100644
--- a/llama_stack/core/build.py
+++ b/llama_stack/core/build.py
@@ -91,7 +91,7 @@ def get_provider_dependencies(
 
 
 def print_pip_install_help(config: BuildConfig):
-    normal_deps, special_deps = get_provider_dependencies(config)
+    normal_deps, special_deps, _ = get_provider_dependencies(config)
 
     cprint(
         f"Please install needed dependencies using the following commands:\n\nuv pip install {' '.join(normal_deps)}",
diff --git a/llama_stack/core/routers/inference.py b/llama_stack/core/routers/inference.py
index 52581cc9d..6a3f07247 100644
--- a/llama_stack/core/routers/inference.py
+++ b/llama_stack/core/routers/inference.py
@@ -65,7 +65,7 @@ from llama_stack.providers.datatypes import HealthResponse, HealthStatus, Routin
 from llama_stack.providers.utils.inference.inference_store import InferenceStore
 from llama_stack.providers.utils.telemetry.tracing import get_current_span
 
-logger = get_logger(name=__name__, category="core")
+logger = get_logger(name=__name__, category="inference")
 
 
 class InferenceRouter(Inference):
@@ -854,4 +854,5 @@ class InferenceRouter(Inference):
                     model=model.identifier,
                     object="chat.completion",
                 )
+                logger.debug(f"InferenceRouter.completion_response: {final_response}")
                 await self.store.store_chat_completion(final_response, messages)
diff --git a/llama_stack/core/routing_tables/models.py b/llama_stack/core/routing_tables/models.py
index c76619271..34c431e00 100644
--- a/llama_stack/core/routing_tables/models.py
+++ b/llama_stack/core/routing_tables/models.py
@@ -63,6 +63,8 @@ class ModelsRoutingTable(CommonRoutingTableImpl, Models):
 
     async def get_provider_impl(self, model_id: str) -> Any:
         model = await lookup_model(self, model_id)
+        if model.provider_id not in self.impls_by_provider_id:
+            raise ValueError(f"Provider {model.provider_id} not found in the routing table")
         return self.impls_by_provider_id[model.provider_id]
 
     async def register_model(
diff --git a/llama_stack/log.py b/llama_stack/log.py
index 0a2d63ef6..7507aface 100644
--- a/llama_stack/log.py
+++ b/llama_stack/log.py
@@ -32,6 +32,7 @@ CATEGORIES = [
     "tools",
     "client",
     "telemetry",
+    "openai_responses",
 ]
 
 # Initialize category levels with default level
diff --git a/llama_stack/models/llama/llama3/chat_format.py b/llama_stack/models/llama/llama3/chat_format.py
index 0a973cf0c..1f88a1699 100644
--- a/llama_stack/models/llama/llama3/chat_format.py
+++ b/llama_stack/models/llama/llama3/chat_format.py
@@ -236,6 +236,7 @@ class ChatFormat:
                     arguments_json=json.dumps(tool_arguments),
                 )
             )
+            content = ""
 
         return RawMessage(
             role="assistant",
diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
index b98ca114f..347954908 100644
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
@@ -488,8 +488,12 @@ class OpenAIResponsesImpl:
             # Convert collected chunks to complete response
             if chat_response_tool_calls:
                 tool_calls = [chat_response_tool_calls[i] for i in sorted(chat_response_tool_calls.keys())]
+
+                # when there are tool calls, we need to clear the content
+                chat_response_content = []
             else:
                 tool_calls = None
+
             assistant_message = OpenAIAssistantMessageParam(
                 content="".join(chat_response_content),
                 tool_calls=tool_calls,
diff --git a/llama_stack/providers/inline/vector_io/faiss/faiss.py b/llama_stack/providers/inline/vector_io/faiss/faiss.py
index 5a063592c..af61da59b 100644
--- a/llama_stack/providers/inline/vector_io/faiss/faiss.py
+++ b/llama_stack/providers/inline/vector_io/faiss/faiss.py
@@ -33,6 +33,7 @@ from llama_stack.providers.utils.kvstore import kvstore_impl
 from llama_stack.providers.utils.kvstore.api import KVStore
 from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIVectorStoreMixin
 from llama_stack.providers.utils.memory.vector_store import (
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -128,11 +129,12 @@ class FaissIndex(EmbeddingIndex):
         # Save updated index
         await self._save_index()
 
-    async def delete_chunk(self, chunk_id: str) -> None:
-        if chunk_id not in self.chunk_ids:
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
+        chunk_ids = [c.chunk_id for c in chunks_for_deletion]
+        if not set(chunk_ids).issubset(self.chunk_ids):
             return
 
-        async with self.chunk_id_lock:
+        def remove_chunk(chunk_id: str):
             index = self.chunk_ids.index(chunk_id)
             self.index.remove_ids(np.array([index]))
 
@@ -146,6 +148,10 @@ class FaissIndex(EmbeddingIndex):
             self.chunk_by_index = new_chunk_by_index
             self.chunk_ids.pop(index)
 
+        async with self.chunk_id_lock:
+            for chunk_id in chunk_ids:
+                remove_chunk(chunk_id)
+
         await self._save_index()
 
     async def query_vector(
@@ -297,8 +303,7 @@ class FaissVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolPr
 
         return await index.query_chunks(query, params)
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
-        """Delete a chunk from a faiss index"""
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
+        """Delete chunks from a faiss index"""
         faiss_index = self.cache[store_id].index
-        for chunk_id in chunk_ids:
-            await faiss_index.delete_chunk(chunk_id)
+        await faiss_index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/inline/vector_io/sqlite_vec/sqlite_vec.py b/llama_stack/providers/inline/vector_io/sqlite_vec/sqlite_vec.py
index 1fff7b484..cc1982f3b 100644
--- a/llama_stack/providers/inline/vector_io/sqlite_vec/sqlite_vec.py
+++ b/llama_stack/providers/inline/vector_io/sqlite_vec/sqlite_vec.py
@@ -31,6 +31,7 @@ from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIV
 from llama_stack.providers.utils.memory.vector_store import (
     RERANKER_TYPE_RRF,
     RERANKER_TYPE_WEIGHTED,
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -426,34 +427,36 @@ class SQLiteVecIndex(EmbeddingIndex):
 
         return QueryChunksResponse(chunks=chunks, scores=scores)
 
-    async def delete_chunk(self, chunk_id: str) -> None:
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Remove a chunk from the SQLite vector store."""
+        chunk_ids = [c.chunk_id for c in chunks_for_deletion]
 
-        def _delete_chunk():
+        def _delete_chunks():
             connection = _create_sqlite_connection(self.db_path)
             cur = connection.cursor()
             try:
                 cur.execute("BEGIN TRANSACTION")
 
                 # Delete from metadata table
-                cur.execute(f"DELETE FROM {self.metadata_table} WHERE id = ?", (chunk_id,))
+                placeholders = ",".join("?" * len(chunk_ids))
+                cur.execute(f"DELETE FROM {self.metadata_table} WHERE id IN ({placeholders})", chunk_ids)
 
                 # Delete from vector table
-                cur.execute(f"DELETE FROM {self.vector_table} WHERE id = ?", (chunk_id,))
+                cur.execute(f"DELETE FROM {self.vector_table} WHERE id IN ({placeholders})", chunk_ids)
 
                 # Delete from FTS table
-                cur.execute(f"DELETE FROM {self.fts_table} WHERE id = ?", (chunk_id,))
+                cur.execute(f"DELETE FROM {self.fts_table} WHERE id IN ({placeholders})", chunk_ids)
 
                 connection.commit()
             except Exception as e:
                 connection.rollback()
-                logger.error(f"Error deleting chunk {chunk_id}: {e}")
+                logger.error(f"Error deleting chunks: {e}")
                 raise
             finally:
                 cur.close()
                 connection.close()
 
-        await asyncio.to_thread(_delete_chunk)
+        await asyncio.to_thread(_delete_chunks)
 
 
 class SQLiteVecVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolPrivate):
@@ -551,12 +554,10 @@ class SQLiteVecVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtoc
             raise VectorStoreNotFoundError(vector_db_id)
         return await index.query_chunks(query, params)
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
-        """Delete a chunk from a sqlite_vec index."""
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
+        """Delete chunks from a sqlite_vec index."""
         index = await self._get_and_cache_vector_db_index(store_id)
         if not index:
             raise VectorStoreNotFoundError(store_id)
 
-        for chunk_id in chunk_ids:
-            # Use the index's delete_chunk method
-            await index.index.delete_chunk(chunk_id)
+        await index.index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/registry/vector_io.py b/llama_stack/providers/registry/vector_io.py
index ed170b508..70148eb15 100644
--- a/llama_stack/providers/registry/vector_io.py
+++ b/llama_stack/providers/registry/vector_io.py
@@ -342,6 +342,7 @@ See [Chroma's documentation](https://docs.trychroma.com/docs/overview/introducti
 """,
             ),
             api_dependencies=[Api.inference],
+            optional_api_dependencies=[Api.files],
         ),
         InlineProviderSpec(
             api=Api.vector_io,
@@ -350,6 +351,7 @@ See [Chroma's documentation](https://docs.trychroma.com/docs/overview/introducti
             module="llama_stack.providers.inline.vector_io.chroma",
             config_class="llama_stack.providers.inline.vector_io.chroma.ChromaVectorIOConfig",
             api_dependencies=[Api.inference],
+            optional_api_dependencies=[Api.files],
             description="""
 [Chroma](https://www.trychroma.com/) is an inline and remote vector
 database provider for Llama Stack. It allows you to store and query vectors directly within a Chroma database.
@@ -464,6 +466,7 @@ See [Weaviate's documentation](https://weaviate.io/developers/weaviate) for more
 """,
             ),
             api_dependencies=[Api.inference],
+            optional_api_dependencies=[Api.files],
         ),
         InlineProviderSpec(
             api=Api.vector_io,
@@ -731,6 +734,7 @@ For more details on TLS configuration, refer to the [TLS setup guide](https://mi
 """,
             ),
             api_dependencies=[Api.inference],
+            optional_api_dependencies=[Api.files],
         ),
         InlineProviderSpec(
             api=Api.vector_io,
diff --git a/llama_stack/providers/remote/inference/fireworks/fireworks.py b/llama_stack/providers/remote/inference/fireworks/fireworks.py
index ca4c7b578..bd86f7238 100644
--- a/llama_stack/providers/remote/inference/fireworks/fireworks.py
+++ b/llama_stack/providers/remote/inference/fireworks/fireworks.py
@@ -235,6 +235,7 @@ class FireworksInferenceAdapter(ModelRegistryHelper, Inference, NeedsRequestProv
 
         llama_model = self.get_llama_model(request.model)
         if isinstance(request, ChatCompletionRequest):
+            # TODO: tools are never added to the request, so we need to add them here
             if media_present or not llama_model:
                 input_dict["messages"] = [
                     await convert_message_to_openai_dict(m, download=True) for m in request.messages
@@ -378,6 +379,7 @@ class FireworksInferenceAdapter(ModelRegistryHelper, Inference, NeedsRequestProv
         # Fireworks chat completions OpenAI-compatible API does not support
         # tool calls properly.
         llama_model = self.get_llama_model(model_obj.provider_resource_id)
+
         if llama_model:
             return await OpenAIChatCompletionToLlamaStackMixin.openai_chat_completion(
                 self,
@@ -431,4 +433,5 @@ class FireworksInferenceAdapter(ModelRegistryHelper, Inference, NeedsRequestProv
             user=user,
         )
 
+        logger.debug(f"fireworks params: {params}")
         return await self._get_openai_client().chat.completions.create(model=model_obj.provider_resource_id, **params)
diff --git a/llama_stack/providers/remote/vector_io/chroma/chroma.py b/llama_stack/providers/remote/vector_io/chroma/chroma.py
index 26aeaedfb..8f252711b 100644
--- a/llama_stack/providers/remote/vector_io/chroma/chroma.py
+++ b/llama_stack/providers/remote/vector_io/chroma/chroma.py
@@ -26,6 +26,7 @@ from llama_stack.providers.utils.kvstore import kvstore_impl
 from llama_stack.providers.utils.kvstore.api import KVStore
 from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIVectorStoreMixin
 from llama_stack.providers.utils.memory.vector_store import (
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -115,8 +116,10 @@ class ChromaIndex(EmbeddingIndex):
     ) -> QueryChunksResponse:
         raise NotImplementedError("Keyword search is not supported in Chroma")
 
-    async def delete_chunk(self, chunk_id: str) -> None:
-        raise NotImplementedError("delete_chunk is not supported in Chroma")
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
+        """Delete a single chunk from the Chroma collection by its ID."""
+        ids = [f"{chunk.document_id}:{chunk.chunk_id}" for chunk in chunks_for_deletion]
+        await maybe_await(self.collection.delete(ids=ids))
 
     async def query_hybrid(
         self,
@@ -144,6 +147,7 @@ class ChromaVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolP
         self.cache = {}
         self.kvstore: KVStore | None = None
         self.vector_db_store = None
+        self.files_api = files_api
 
     async def initialize(self) -> None:
         self.kvstore = await kvstore_impl(self.config.kvstore)
@@ -227,5 +231,10 @@ class ChromaVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolP
         self.cache[vector_db_id] = index
         return index
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
-        raise NotImplementedError("OpenAI Vector Stores API is not supported in Chroma")
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
+        """Delete chunks from a Chroma vector store."""
+        index = await self._get_and_cache_vector_db_index(store_id)
+        if not index:
+            raise ValueError(f"Vector DB {store_id} not found")
+
+        await index.index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/remote/vector_io/milvus/milvus.py b/llama_stack/providers/remote/vector_io/milvus/milvus.py
index b09edb65c..0eaae81b3 100644
--- a/llama_stack/providers/remote/vector_io/milvus/milvus.py
+++ b/llama_stack/providers/remote/vector_io/milvus/milvus.py
@@ -28,6 +28,7 @@ from llama_stack.providers.utils.kvstore.api import KVStore
 from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIVectorStoreMixin
 from llama_stack.providers.utils.memory.vector_store import (
     RERANKER_TYPE_WEIGHTED,
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -287,14 +288,17 @@ class MilvusIndex(EmbeddingIndex):
 
         return QueryChunksResponse(chunks=filtered_chunks, scores=filtered_scores)
 
-    async def delete_chunk(self, chunk_id: str) -> None:
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Remove a chunk from the Milvus collection."""
+        chunk_ids = [c.chunk_id for c in chunks_for_deletion]
         try:
+            # Use IN clause with square brackets and single quotes for VARCHAR field
+            chunk_ids_str = ", ".join(f"'{chunk_id}'" for chunk_id in chunk_ids)
             await asyncio.to_thread(
-                self.client.delete, collection_name=self.collection_name, filter=f'chunk_id == "{chunk_id}"'
+                self.client.delete, collection_name=self.collection_name, filter=f"chunk_id in [{chunk_ids_str}]"
             )
         except Exception as e:
-            logger.error(f"Error deleting chunk {chunk_id} from Milvus collection {self.collection_name}: {e}")
+            logger.error(f"Error deleting chunks from Milvus collection {self.collection_name}: {e}")
             raise
 
 
@@ -420,12 +424,10 @@ class MilvusVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolP
 
         return await index.query_chunks(query, params)
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Delete a chunk from a milvus vector store."""
         index = await self._get_and_cache_vector_db_index(store_id)
         if not index:
             raise VectorStoreNotFoundError(store_id)
 
-        for chunk_id in chunk_ids:
-            # Use the index's delete_chunk method
-            await index.index.delete_chunk(chunk_id)
+        await index.index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/remote/vector_io/pgvector/pgvector.py b/llama_stack/providers/remote/vector_io/pgvector/pgvector.py
index b1645ac5a..d2a5d910b 100644
--- a/llama_stack/providers/remote/vector_io/pgvector/pgvector.py
+++ b/llama_stack/providers/remote/vector_io/pgvector/pgvector.py
@@ -27,6 +27,7 @@ from llama_stack.providers.utils.kvstore import kvstore_impl
 from llama_stack.providers.utils.kvstore.api import KVStore
 from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIVectorStoreMixin
 from llama_stack.providers.utils.memory.vector_store import (
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -163,10 +164,11 @@ class PGVectorIndex(EmbeddingIndex):
         with self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
             cur.execute(f"DROP TABLE IF EXISTS {self.table_name}")
 
-    async def delete_chunk(self, chunk_id: str) -> None:
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Remove a chunk from the PostgreSQL table."""
+        chunk_ids = [c.chunk_id for c in chunks_for_deletion]
         with self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
-            cur.execute(f"DELETE FROM {self.table_name} WHERE id = %s", (chunk_id,))
+            cur.execute(f"DELETE FROM {self.table_name} WHERE id = ANY(%s)", (chunk_ids,))
 
 
 class PGVectorVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolPrivate):
@@ -275,12 +277,10 @@ class PGVectorVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtoco
         self.cache[vector_db_id] = VectorDBWithIndex(vector_db, index, self.inference_api)
         return self.cache[vector_db_id]
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Delete a chunk from a PostgreSQL vector store."""
         index = await self._get_and_cache_vector_db_index(store_id)
         if not index:
             raise VectorStoreNotFoundError(store_id)
 
-        for chunk_id in chunk_ids:
-            # Use the index's delete_chunk method
-            await index.index.delete_chunk(chunk_id)
+        await index.index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/remote/vector_io/qdrant/qdrant.py b/llama_stack/providers/remote/vector_io/qdrant/qdrant.py
index 144da0f4f..018015780 100644
--- a/llama_stack/providers/remote/vector_io/qdrant/qdrant.py
+++ b/llama_stack/providers/remote/vector_io/qdrant/qdrant.py
@@ -29,6 +29,7 @@ from llama_stack.providers.inline.vector_io.qdrant import QdrantVectorIOConfig a
 from llama_stack.providers.utils.kvstore import KVStore, kvstore_impl
 from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIVectorStoreMixin
 from llama_stack.providers.utils.memory.vector_store import (
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -88,15 +89,16 @@ class QdrantIndex(EmbeddingIndex):
 
         await self.client.upsert(collection_name=self.collection_name, points=points)
 
-    async def delete_chunk(self, chunk_id: str) -> None:
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Remove a chunk from the Qdrant collection."""
+        chunk_ids = [convert_id(c.chunk_id) for c in chunks_for_deletion]
         try:
             await self.client.delete(
                 collection_name=self.collection_name,
-                points_selector=models.PointIdsList(points=[convert_id(chunk_id)]),
+                points_selector=models.PointIdsList(points=chunk_ids),
             )
         except Exception as e:
-            log.error(f"Error deleting chunk {chunk_id} from Qdrant collection {self.collection_name}: {e}")
+            log.error(f"Error deleting chunks from Qdrant collection {self.collection_name}: {e}")
             raise
 
     async def query_vector(self, embedding: NDArray, k: int, score_threshold: float) -> QueryChunksResponse:
@@ -264,12 +266,14 @@ class QdrantVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolP
     ) -> VectorStoreFileObject:
         # Qdrant doesn't allow multiple clients to access the same storage path simultaneously.
         async with self._qdrant_lock:
-            await super().openai_attach_file_to_vector_store(vector_store_id, file_id, attributes, chunking_strategy)
+            return await super().openai_attach_file_to_vector_store(
+                vector_store_id, file_id, attributes, chunking_strategy
+            )
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         """Delete chunks from a Qdrant vector store."""
         index = await self._get_and_cache_vector_db_index(store_id)
         if not index:
             raise ValueError(f"Vector DB {store_id} not found")
-        for chunk_id in chunk_ids:
-            await index.index.delete_chunk(chunk_id)
+
+        await index.index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/remote/vector_io/weaviate/weaviate.py b/llama_stack/providers/remote/vector_io/weaviate/weaviate.py
index 11da8902c..966724848 100644
--- a/llama_stack/providers/remote/vector_io/weaviate/weaviate.py
+++ b/llama_stack/providers/remote/vector_io/weaviate/weaviate.py
@@ -26,6 +26,7 @@ from llama_stack.providers.utils.memory.openai_vector_store_mixin import (
     OpenAIVectorStoreMixin,
 )
 from llama_stack.providers.utils.memory.vector_store import (
+    ChunkForDeletion,
     EmbeddingIndex,
     VectorDBWithIndex,
 )
@@ -67,6 +68,7 @@ class WeaviateIndex(EmbeddingIndex):
             data_objects.append(
                 wvc.data.DataObject(
                     properties={
+                        "chunk_id": chunk.chunk_id,
                         "chunk_content": chunk.model_dump_json(),
                     },
                     vector=embeddings[i].tolist(),
@@ -79,10 +81,11 @@ class WeaviateIndex(EmbeddingIndex):
         # TODO: make this async friendly
         collection.data.insert_many(data_objects)
 
-    async def delete_chunk(self, chunk_id: str) -> None:
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         sanitized_collection_name = sanitize_collection_name(self.collection_name, weaviate_format=True)
         collection = self.client.collections.get(sanitized_collection_name)
-        collection.data.delete_many(where=Filter.by_property("id").contains_any([chunk_id]))
+        chunk_ids = [chunk.chunk_id for chunk in chunks_for_deletion]
+        collection.data.delete_many(where=Filter.by_property("chunk_id").contains_any(chunk_ids))
 
     async def query_vector(self, embedding: NDArray, k: int, score_threshold: float) -> QueryChunksResponse:
         sanitized_collection_name = sanitize_collection_name(self.collection_name, weaviate_format=True)
@@ -307,10 +310,10 @@ class WeaviateVectorIOAdapter(
 
         return await index.query_chunks(query, params)
 
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
         sanitized_collection_name = sanitize_collection_name(store_id, weaviate_format=True)
         index = await self._get_and_cache_vector_db_index(sanitized_collection_name)
         if not index:
             raise ValueError(f"Vector DB {sanitized_collection_name} not found")
 
-        await index.delete(chunk_ids)
+        await index.index.delete_chunks(chunks_for_deletion)
diff --git a/llama_stack/providers/utils/memory/openai_vector_store_mixin.py b/llama_stack/providers/utils/memory/openai_vector_store_mixin.py
index 7b6e69df1..120d0d4fc 100644
--- a/llama_stack/providers/utils/memory/openai_vector_store_mixin.py
+++ b/llama_stack/providers/utils/memory/openai_vector_store_mixin.py
@@ -6,7 +6,6 @@
 
 import asyncio
 import json
-import logging
 import mimetypes
 import time
 import uuid
@@ -37,10 +36,15 @@ from llama_stack.apis.vector_io import (
     VectorStoreSearchResponse,
     VectorStoreSearchResponsePage,
 )
+from llama_stack.log import get_logger
 from llama_stack.providers.utils.kvstore.api import KVStore
-from llama_stack.providers.utils.memory.vector_store import content_from_data_and_mime_type, make_overlapped_chunks
+from llama_stack.providers.utils.memory.vector_store import (
+    ChunkForDeletion,
+    content_from_data_and_mime_type,
+    make_overlapped_chunks,
+)
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__, category="vector_io")
 
 # Constants for OpenAI vector stores
 CHUNK_MULTIPLIER = 5
@@ -154,8 +158,8 @@ class OpenAIVectorStoreMixin(ABC):
         self.openai_vector_stores = await self._load_openai_vector_stores()
 
     @abstractmethod
-    async def delete_chunks(self, store_id: str, chunk_ids: list[str]) -> None:
-        """Delete a chunk from a vector store."""
+    async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None:
+        """Delete chunks from a vector store."""
         pass
 
     @abstractmethod
@@ -614,7 +618,7 @@ class OpenAIVectorStoreMixin(ABC):
                 )
                 vector_store_file_object.status = "completed"
         except Exception as e:
-            logger.error(f"Error attaching file to vector store: {e}")
+            logger.exception("Error attaching file to vector store")
             vector_store_file_object.status = "failed"
             vector_store_file_object.last_error = VectorStoreFileLastError(
                 code="server_error",
@@ -767,7 +771,21 @@ class OpenAIVectorStoreMixin(ABC):
 
         dict_chunks = await self._load_openai_vector_store_file_contents(vector_store_id, file_id)
         chunks = [Chunk.model_validate(c) for c in dict_chunks]
-        await self.delete_chunks(vector_store_id, [str(c.chunk_id) for c in chunks if c.chunk_id])
+
+        # Create ChunkForDeletion objects with both chunk_id and document_id
+        chunks_for_deletion = []
+        for c in chunks:
+            if c.chunk_id:
+                document_id = c.metadata.get("document_id") or (
+                    c.chunk_metadata.document_id if c.chunk_metadata else None
+                )
+                if document_id:
+                    chunks_for_deletion.append(ChunkForDeletion(chunk_id=str(c.chunk_id), document_id=document_id))
+                else:
+                    logger.warning(f"Chunk {c.chunk_id} has no document_id, skipping deletion")
+
+        if chunks_for_deletion:
+            await self.delete_chunks(vector_store_id, chunks_for_deletion)
 
         store_info = self.openai_vector_stores[vector_store_id].copy()
 
diff --git a/llama_stack/providers/utils/memory/vector_store.py b/llama_stack/providers/utils/memory/vector_store.py
index bb9002f30..6ae5bb521 100644
--- a/llama_stack/providers/utils/memory/vector_store.py
+++ b/llama_stack/providers/utils/memory/vector_store.py
@@ -16,6 +16,7 @@ from urllib.parse import unquote
 import httpx
 import numpy as np
 from numpy.typing import NDArray
+from pydantic import BaseModel
 
 from llama_stack.apis.common.content_types import (
     URL,
@@ -34,6 +35,18 @@ from llama_stack.providers.utils.vector_io.vector_utils import generate_chunk_id
 
 log = logging.getLogger(__name__)
 
+
+class ChunkForDeletion(BaseModel):
+    """Information needed to delete a chunk from a vector store.
+
+    :param chunk_id: The ID of the chunk to delete
+    :param document_id: The ID of the document this chunk belongs to
+    """
+
+    chunk_id: str
+    document_id: str
+
+
 # Constants for reranker types
 RERANKER_TYPE_RRF = "rrf"
 RERANKER_TYPE_WEIGHTED = "weighted"
@@ -232,7 +245,7 @@ class EmbeddingIndex(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    async def delete_chunk(self, chunk_id: str):
+    async def delete_chunks(self, chunks_for_deletion: list[ChunkForDeletion]):
         raise NotImplementedError()
 
     @abstractmethod
diff --git a/tests/common/mcp.py b/tests/common/mcp.py
index 775e38295..d05ac39c6 100644
--- a/tests/common/mcp.py
+++ b/tests/common/mcp.py
@@ -16,13 +16,10 @@ MCP_TOOLGROUP_ID = "mcp::localmcp"
 
 def default_tools():
     """Default tools for backward compatibility."""
-    from mcp import types
     from mcp.server.fastmcp import Context
 
-    async def greet_everyone(
-        url: str, ctx: Context
-    ) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
-        return [types.TextContent(type="text", text="Hello, world!")]
+    async def greet_everyone(url: str, ctx: Context) -> str:
+        return "Hello, world!"
 
     async def get_boiling_point(liquid_name: str, celsius: bool = True) -> int:
         """
@@ -45,7 +42,6 @@ def default_tools():
 
 def dependency_tools():
     """Tools with natural dependencies for multi-turn testing."""
-    from mcp import types
     from mcp.server.fastmcp import Context
 
     async def get_user_id(username: str, ctx: Context) -> str:
@@ -106,7 +102,7 @@ def dependency_tools():
         else:
             access = "no"
 
-        return [types.TextContent(type="text", text=access)]
+        return access
 
     async def get_experiment_id(experiment_name: str, ctx: Context) -> str:
         """
@@ -245,7 +241,6 @@ def make_mcp_server(required_auth_token: str | None = None, tools: dict[str, Cal
     try:
         yield {"server_url": server_url}
     finally:
-        print("Telling SSE server to exit")
         server_instance.should_exit = True
         time.sleep(0.5)
 
@@ -269,4 +264,3 @@ def make_mcp_server(required_auth_token: str | None = None, tools: dict[str, Cal
 
         AppStatus.should_exit = False
         AppStatus.should_exit_event = None
-        print("SSE server exited")
diff --git a/tests/integration/fixtures/common.py b/tests/integration/fixtures/common.py
index c91391f19..0b7132d71 100644
--- a/tests/integration/fixtures/common.py
+++ b/tests/integration/fixtures/common.py
@@ -270,7 +270,7 @@ def openai_client(client_with_models):
 
 @pytest.fixture(params=["openai_client", "client_with_models"])
 def compat_client(request, client_with_models):
-    if isinstance(client_with_models, LlamaStackAsLibraryClient):
+    if request.param == "openai_client" and isinstance(client_with_models, LlamaStackAsLibraryClient):
         # OpenAI client expects a server, so unless we also rewrite OpenAI client's requests
         # to go via the Stack library client (which itself rewrites requests to be served inline),
         # we cannot do this.
diff --git a/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml b/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml
index 6db0dd970..353a64291 100644
--- a/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml
+++ b/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml
@@ -137,7 +137,7 @@ test_response_multi_turn_tool_execution:
         server_url: "<FILLED_BY_TEST_RUNNER>"
       output: "yes"
     - case_id: "experiment_results_lookup"
-      input: "I need to get the results for the 'boiling_point' experiment. First, get the experiment ID for 'boiling_point', then use that ID to get the experiment results. Tell me what you found."
+      input: "I need to get the results for the 'boiling_point' experiment. First, get the experiment ID for 'boiling_point', then use that ID to get the experiment results. Tell me the boiling point in Celsius."
       tools:
       - type: mcp
         server_label: "localmcp"
@@ -149,7 +149,7 @@ test_response_multi_turn_tool_execution_streaming:
   test_params:
     case:
     - case_id: "user_permissions_workflow"
-      input: "Help me with this security check: First, get the user ID for 'charlie', then get the permissions for that user ID, and finally check if that user can access 'secret_file.txt'. Stream your progress as you work through each step."
+      input: "Help me with this security check: First, get the user ID for 'charlie', then get the permissions for that user ID, and finally check if that user can access 'secret_file.txt'. Stream your progress as you work through each step. Return only one tool call per step. Summarize the final result with a single 'yes' or 'no' response."
       tools:
       - type: mcp
         server_label: "localmcp"
@@ -157,7 +157,7 @@ test_response_multi_turn_tool_execution_streaming:
       stream: true
       output: "no"
     - case_id: "experiment_analysis_streaming"
-      input: "I need a complete analysis: First, get the experiment ID for 'chemical_reaction', then get the results for that experiment, and tell me if the yield was above 80%. Please stream your analysis process."
+      input: "I need a complete analysis: First, get the experiment ID for 'chemical_reaction', then get the results for that experiment, and tell me if the yield was above 80%. Return only one tool call per step.  Please stream your analysis process."
       tools:
       - type: mcp
         server_label: "localmcp"
diff --git a/tests/integration/non_ci/responses/test_responses.py b/tests/integration/non_ci/responses/test_responses.py
index 4f4f27d7f..39d00f328 100644
--- a/tests/integration/non_ci/responses/test_responses.py
+++ b/tests/integration/non_ci/responses/test_responses.py
@@ -363,6 +363,9 @@ def test_response_non_streaming_file_search_empty_vector_store(request, compat_c
     ids=case_id_generator,
 )
 def test_response_non_streaming_mcp_tool(request, compat_client, text_model_id, case):
+    if not isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("in-process MCP server is only supported in library client")
+
     with make_mcp_server() as mcp_server_info:
         tools = case["tools"]
         for tool in tools:
@@ -485,8 +488,11 @@ def test_response_non_streaming_multi_turn_image(request, compat_client, text_mo
     responses_test_cases["test_response_multi_turn_tool_execution"]["test_params"]["case"],
     ids=case_id_generator,
 )
-def test_response_non_streaming_multi_turn_tool_execution(request, compat_client, text_model_id, case):
+def test_response_non_streaming_multi_turn_tool_execution(compat_client, text_model_id, case):
     """Test multi-turn tool execution where multiple MCP tool calls are performed in sequence."""
+    if not isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("in-process MCP server is only supported in library client")
+
     with make_mcp_server(tools=dependency_tools()) as mcp_server_info:
         tools = case["tools"]
         # Replace the placeholder URL with the actual server URL
@@ -541,8 +547,11 @@ def test_response_non_streaming_multi_turn_tool_execution(request, compat_client
     responses_test_cases["test_response_multi_turn_tool_execution_streaming"]["test_params"]["case"],
     ids=case_id_generator,
 )
-async def test_response_streaming_multi_turn_tool_execution(request, compat_client, text_model_id, case):
+def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_id, case):
     """Test streaming multi-turn tool execution where multiple MCP tool calls are performed in sequence."""
+    if not isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("in-process MCP server is only supported in library client")
+
     with make_mcp_server(tools=dependency_tools()) as mcp_server_info:
         tools = case["tools"]
         # Replace the placeholder URL with the actual server URL
@@ -634,7 +643,7 @@ async def test_response_streaming_multi_turn_tool_execution(request, compat_clie
         },
     ],
 )
-def test_response_text_format(request, compat_client, text_model_id, text_format):
+def test_response_text_format(compat_client, text_model_id, text_format):
     if isinstance(compat_client, LlamaStackAsLibraryClient):
         pytest.skip("Responses API text format is not yet supported in library client.")
 
@@ -653,7 +662,7 @@ def test_response_text_format(request, compat_client, text_model_id, text_format
 
 
 @pytest.fixture
-def vector_store_with_filtered_files(request, compat_client, text_model_id, tmp_path_factory):
+def vector_store_with_filtered_files(compat_client, text_model_id, tmp_path_factory):
     """Create a vector store with multiple files that have different attributes for filtering tests."""
     if isinstance(compat_client, LlamaStackAsLibraryClient):
         pytest.skip("Responses API file search is not yet supported in library client.")
diff --git a/tests/integration/vector_io/test_openai_vector_stores.py b/tests/integration/vector_io/test_openai_vector_stores.py
index 3212a7568..7ccca9077 100644
--- a/tests/integration/vector_io/test_openai_vector_stores.py
+++ b/tests/integration/vector_io/test_openai_vector_stores.py
@@ -9,10 +9,11 @@ import time
 from io import BytesIO
 
 import pytest
-from llama_stack_client import BadRequestError, LlamaStackClient
+from llama_stack_client import BadRequestError
 from openai import BadRequestError as OpenAIBadRequestError
 
 from llama_stack.apis.vector_io import Chunk
+from llama_stack.core.library_client import LlamaStackAsLibraryClient
 
 logger = logging.getLogger(__name__)
 
@@ -475,9 +476,6 @@ def test_openai_vector_store_attach_file(compat_client_with_empty_stores, client
     """Test OpenAI vector store attach file."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files attach is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store
@@ -526,9 +524,6 @@ def test_openai_vector_store_attach_files_on_creation(compat_client_with_empty_s
     """Test OpenAI vector store attach files on creation."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files attach is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create some files and attach them to the vector store
@@ -582,9 +577,6 @@ def test_openai_vector_store_list_files(compat_client_with_empty_stores, client_
     """Test OpenAI vector store list files."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files list is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store
@@ -597,16 +589,20 @@ def test_openai_vector_store_list_files(compat_client_with_empty_stores, client_
             file_buffer.name = f"openai_test_{i}.txt"
             file = compat_client.files.create(file=file_buffer, purpose="assistants")
 
-        compat_client.vector_stores.files.create(
+        response = compat_client.vector_stores.files.create(
             vector_store_id=vector_store.id,
             file_id=file.id,
         )
+        assert response is not None
+        assert response.status == "completed", (
+            f"Failed to attach file {file.id} to vector store {vector_store.id}: {response=}"
+        )
         file_ids.append(file.id)
 
     files_list = compat_client.vector_stores.files.list(vector_store_id=vector_store.id)
     assert files_list
     assert files_list.object == "list"
-    assert files_list.data
+    assert files_list.data is not None
     assert not files_list.has_more
     assert len(files_list.data) == 3
     assert set(file_ids) == {file.id for file in files_list.data}
@@ -642,12 +638,13 @@ def test_openai_vector_store_list_files_invalid_vector_store(compat_client_with_
     """Test OpenAI vector store list files with invalid vector store ID."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files list is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
+    if isinstance(compat_client, LlamaStackAsLibraryClient):
+        errors = ValueError
+    else:
+        errors = (BadRequestError, OpenAIBadRequestError)
 
-    with pytest.raises((BadRequestError, OpenAIBadRequestError)):
+    with pytest.raises(errors):
         compat_client.vector_stores.files.list(vector_store_id="abc123")
 
 
@@ -655,9 +652,6 @@ def test_openai_vector_store_retrieve_file_contents(compat_client_with_empty_sto
     """Test OpenAI vector store retrieve file contents."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files retrieve contents is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store
@@ -685,9 +679,15 @@ def test_openai_vector_store_retrieve_file_contents(compat_client_with_empty_sto
         file_id=file.id,
     )
 
-    assert file_contents
-    assert file_contents.content[0]["type"] == "text"
-    assert file_contents.content[0]["text"] == test_content.decode("utf-8")
+    assert file_contents is not None
+    assert len(file_contents.content) == 1
+    content = file_contents.content[0]
+
+    # llama-stack-client returns a model, openai-python is a badboy and returns a dict
+    if not isinstance(content, dict):
+        content = content.model_dump()
+    assert content["type"] == "text"
+    assert content["text"] == test_content.decode("utf-8")
     assert file_contents.filename == file_name
     assert file_contents.attributes == attributes
 
@@ -696,9 +696,6 @@ def test_openai_vector_store_delete_file(compat_client_with_empty_stores, client
     """Test OpenAI vector store delete file."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files list is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store
@@ -751,9 +748,6 @@ def test_openai_vector_store_delete_file_removes_from_vector_store(compat_client
     """Test OpenAI vector store delete file removes from vector store."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files attach is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store
@@ -792,9 +786,6 @@ def test_openai_vector_store_update_file(compat_client_with_empty_stores, client
     """Test OpenAI vector store update file."""
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files update is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store
@@ -840,9 +831,6 @@ def test_create_vector_store_files_duplicate_vector_store_name(compat_client_wit
     """
     skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
 
-    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
-        pytest.skip("Vector Store Files create is not yet supported with LlamaStackClient")
-
     compat_client = compat_client_with_empty_stores
 
     # Create a vector store with files

From 6358d0a47832185c5484ede73d97fff450db10f5 Mon Sep 17 00:00:00 2001
From: Kelly Brown <86735520+kelbrown20@users.noreply.github.com>
Date: Tue, 12 Aug 2025 19:17:03 -0400
Subject: [PATCH 10/45] docs: reorganize contributor guide (#3110)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

**Description:**
Restructures contribution guide and move some sections into categories

<img width="1399" height="527" alt="Screenshot 2025-08-12 at 9 28 44 AM"
src="https://github.com/user-attachments/assets/404e23b4-0001-4174-b662-593e0173ef7d"
/>
---
 CONTRIBUTING.md                   | 171 ++++++++++++++----------------
 docs/source/contributing/index.md |  22 ++--
 2 files changed, 92 insertions(+), 101 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 066fcecf0..c81e9e7b1 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,13 +1,82 @@
-# Contributing to Llama-Stack
+# Contributing to Llama Stack
 We want to make contributing to this project as easy and transparent as
 possible.
 
+## Set up your development environment
+
+We use [uv](https://github.com/astral-sh/uv) to manage python dependencies and virtual environments.
+You can install `uv` by following this [guide](https://docs.astral.sh/uv/getting-started/installation/).
+
+You can install the dependencies by running:
+
+```bash
+cd llama-stack
+uv sync --group dev
+uv pip install -e .
+source .venv/bin/activate
+```
+
+```{note}
+You can use a specific version of Python with `uv` by adding the `--python <version>` flag (e.g. `--python 3.12`).
+Otherwise, `uv` will automatically select a Python version according to the `requires-python` section of the `pyproject.toml`.
+For more info, see the [uv docs around Python versions](https://docs.astral.sh/uv/concepts/python-versions/).
+```
+
+Note that you can create a dotenv file `.env` that includes necessary environment variables:
+```
+LLAMA_STACK_BASE_URL=http://localhost:8321
+LLAMA_STACK_CLIENT_LOG=debug
+LLAMA_STACK_PORT=8321
+LLAMA_STACK_CONFIG=<provider-name>
+TAVILY_SEARCH_API_KEY=
+BRAVE_SEARCH_API_KEY=
+```
+
+And then use this dotenv file when running client SDK tests via the following:
+```bash
+uv run --env-file .env -- pytest -v tests/integration/inference/test_text_inference.py --text-model=meta-llama/Llama-3.1-8B-Instruct
+```
+
+### Pre-commit Hooks
+
+We use [pre-commit](https://pre-commit.com/) to run linting and formatting checks on your code. You can install the pre-commit hooks by running:
+
+```bash
+uv run pre-commit install
+```
+
+After that, pre-commit hooks will run automatically before each commit.
+
+Alternatively, if you don't want to install the pre-commit hooks, you can run the checks manually by running:
+
+```bash
+uv run pre-commit run --all-files
+```
+
+```{caution}
+Before pushing your changes, make sure that the pre-commit hooks have passed successfully.
+```
+
 ## Discussions -> Issues -> Pull Requests
 
 We actively welcome your pull requests. However, please read the following. This is heavily inspired by [Ghostty](https://github.com/ghostty-org/ghostty/blob/main/CONTRIBUTING.md).
 
 If in doubt, please open a [discussion](https://github.com/meta-llama/llama-stack/discussions); we can always convert that to an issue later.
 
+### Issues
+We use GitHub issues to track public bugs. Please ensure your description is
+clear and has sufficient instructions to be able to reproduce the issue.
+
+Meta has a [bounty program](http://facebook.com/whitehat/info) for the safe
+disclosure of security bugs. In those cases, please go through the process
+outlined on that page and do not file a public issue.
+
+### Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Meta's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
 **I'd like to contribute!**
 
 If you are new to the project, start by looking at the issues tagged with "good first issue". If you're interested
@@ -51,93 +120,15 @@ Please avoid picking up too many issues at once. This helps you stay focused and
 
 Please keep pull requests (PRs) small and focused. If you have a large set of changes, consider splitting them into logically grouped, smaller PRs to facilitate review and testing.
 
-> [!TIP]
-> As a general guideline:
-> - Experienced contributors should try to keep no more than 5 open PRs at a time.
-> - New contributors are encouraged to have only one open PR at a time until they’re familiar with the codebase and process.
-
-## Contributor License Agreement ("CLA")
-In order to accept your pull request, we need you to submit a CLA. You only need
-to do this once to work on any of Meta's open source projects.
-
-Complete your CLA here: <https://code.facebook.com/cla>
-
-## Issues
-We use GitHub issues to track public bugs. Please ensure your description is
-clear and has sufficient instructions to be able to reproduce the issue.
-
-Meta has a [bounty program](http://facebook.com/whitehat/info) for the safe
-disclosure of security bugs. In those cases, please go through the process
-outlined on that page and do not file a public issue.
-
-
-## Set up your development environment
-
-We use [uv](https://github.com/astral-sh/uv) to manage python dependencies and virtual environments.
-You can install `uv` by following this [guide](https://docs.astral.sh/uv/getting-started/installation/).
-
-You can install the dependencies by running:
-
-```bash
-cd llama-stack
-uv sync --group dev
-uv pip install -e .
-source .venv/bin/activate
+```{tip}
+As a general guideline:
+- Experienced contributors should try to keep no more than 5 open PRs at a time.
+- New contributors are encouraged to have only one open PR at a time until they’re familiar with the codebase and process.
 ```
 
-> [!NOTE]
-> You can use a specific version of Python with `uv` by adding the `--python <version>` flag (e.g. `--python 3.12`)
-> Otherwise, `uv` will automatically select a Python version according to the `requires-python` section of the `pyproject.toml`.
-> For more info, see the [uv docs around Python versions](https://docs.astral.sh/uv/concepts/python-versions/).
+## Repository guidelines
 
-Note that you can create a dotenv file `.env` that includes necessary environment variables:
-```
-LLAMA_STACK_BASE_URL=http://localhost:8321
-LLAMA_STACK_CLIENT_LOG=debug
-LLAMA_STACK_PORT=8321
-LLAMA_STACK_CONFIG=<provider-name>
-TAVILY_SEARCH_API_KEY=
-BRAVE_SEARCH_API_KEY=
-```
-
-And then use this dotenv file when running client SDK tests via the following:
-```bash
-uv run --env-file .env -- pytest -v tests/integration/inference/test_text_inference.py --text-model=meta-llama/Llama-3.1-8B-Instruct
-```
-
-## Pre-commit Hooks
-
-We use [pre-commit](https://pre-commit.com/) to run linting and formatting checks on your code. You can install the pre-commit hooks by running:
-
-```bash
-uv run pre-commit install
-```
-
-After that, pre-commit hooks will run automatically before each commit.
-
-Alternatively, if you don't want to install the pre-commit hooks, you can run the checks manually by running:
-
-```bash
-uv run pre-commit run --all-files
-```
-
-> [!CAUTION]
-> Before pushing your changes, make sure that the pre-commit hooks have passed successfully.
-
-## Running tests
-
-You can find the Llama Stack testing documentation [here](https://github.com/meta-llama/llama-stack/blob/main/tests/README.md).
-
-## Adding a new dependency to the project
-
-To add a new dependency to the project, you can use the `uv` command. For example, to add `foo` to the project, you can run:
-
-```bash
-uv add foo
-uv sync
-```
-
-## Coding Style
+### Coding Style
 
 * Comments should provide meaningful insights into the code. Avoid filler comments that simply
   describe the next step, as they create unnecessary clutter, same goes for docstrings.
@@ -159,6 +150,10 @@ uv sync
 * When possible, use keyword arguments only when calling functions.
 * Llama Stack utilizes [custom Exception classes](llama_stack/apis/common/errors.py) for certain Resources that should be used where applicable.
 
+### License
+By contributing to Llama, you agree that your contributions will be licensed
+under the LICENSE file in the root directory of this source tree.
+
 ## Common Tasks
 
 Some tips about common tasks you work on while contributing to Llama Stack:
@@ -210,8 +205,4 @@ If you modify or add new API endpoints, update the API documentation accordingly
 uv run ./docs/openapi_generator/run_openapi_generator.sh
 ```
 
-The generated API documentation will be available in `docs/_static/`. Make sure to review the changes before committing.
-
-## License
-By contributing to Llama, you agree that your contributions will be licensed
-under the LICENSE file in the root directory of this source tree.
+The generated API documentation will be available in `docs/_static/`. Make sure to review the changes before committing.
\ No newline at end of file
diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md
index 79c3861ea..7a3a1c2e2 100644
--- a/docs/source/contributing/index.md
+++ b/docs/source/contributing/index.md
@@ -2,17 +2,6 @@
 ```{include} ../../../CONTRIBUTING.md
 ```
 
-## Testing
-
-See the [Test Page](testing.md) which describes how to test your changes.
-```{toctree}
-:maxdepth: 1
-:hidden:
-:caption: Testing
-
-testing
-```
-
 ## Adding a New Provider
 
 See the [Adding a New API Provider Page](new_api_provider.md) which describes how to add new API providers to the Stack.
@@ -27,3 +16,14 @@ See the [External Provider Page](../providers/external/index.md) which describes
 new_api_provider
 new_vector_database
 ```
+
+## Testing
+
+See the [Test Page](testing.md) which describes how to test your changes.
+```{toctree}
+:maxdepth: 1
+:hidden:
+:caption: Testing
+
+testing
+```
\ No newline at end of file

From fffdab4f5c23b4ac045b898bdb1e69edda3a3498 Mon Sep 17 00:00:00 2001
From: Chacksu <connorhack10@gmail.com>
Date: Wed, 13 Aug 2025 09:18:25 -0400
Subject: [PATCH 11/45] fix: Dell distribution missing kvstore (#3113)

# What does this PR do?

- Added kvstore config to ChromaDB provider config for Dell distribution
similar to [starter
config](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/distributions/starter/run.yaml#L110-L112)
- Fixed
[error](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/inference/_generated/_async_client.py#L3424-L3425)
getting endpoint information by adding `hf-inference` as the provider to
the `AsyncInferenceClient` (TGI client).

## Test Plan
```
export INFERENCE_PORT=8181
export DEH_URL=http://0.0.0.0:$INFERENCE_PORT
export INFERENCE_MODEL=meta-llama/Llama-3.2-3B-Instruct
export CHROMADB_HOST=localhost
export CHROMADB_PORT=8000
export CHROMA_URL=http://$CHROMADB_HOST:$CHROMADB_PORT
export CUDA_VISIBLE_DEVICES=0
export LLAMA_STACK_PORT=8321
export HF_TOKEN=[redacted]

# TGI Server
docker run --rm -it \
  --pull always \
  --network host \
  -v $HOME/.cache/huggingface:/data \
  -e HF_TOKEN=$HF_TOKEN \
  -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
  -p $INFERENCE_PORT:$INFERENCE_PORT \
  --gpus all \
  ghcr.io/huggingface/text-generation-inference:latest \
  --dtype float16 \
  --usage-stats off \
  --sharded false \
  --cuda-memory-fraction 0.8 \
  --model-id meta-llama/Llama-3.2-3B-Instruct \
  --port $INFERENCE_PORT \
  --hostname 0.0.0.0

# Chrome DB
docker run --rm -it \
  --name chromadb \
  --net=host  -p 8000:8000 \
  -v ~/chroma:/chroma/chroma \
  -e IS_PERSISTENT=TRUE \
  -e ANONYMIZED_TELEMETRY=FALSE \
  chromadb/chroma:latest

# Llama Stack
llama stack run dell \
 --port $LLAMA_STACK_PORT \
 --env INFERENCE_MODEL=$INFERENCE_MODEL \
 --env DEH_URL=$DEH_URL \
 --env CHROMA_URL=$CHROMA_URL
```

---------

Co-authored-by: Connor Hack <connorhack@fb.com>
Co-authored-by: Ashwin Bharambe <ashwin.bharambe@gmail.com>
---
 llama_stack/distributions/dell/dell.py              | 8 +++++---
 llama_stack/distributions/dell/run-with-safety.yaml | 5 ++++-
 llama_stack/distributions/dell/run.yaml             | 5 ++++-
 llama_stack/providers/remote/inference/tgi/tgi.py   | 4 +---
 4 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/llama_stack/distributions/dell/dell.py b/llama_stack/distributions/dell/dell.py
index b561ea00e..e3bf0ee03 100644
--- a/llama_stack/distributions/dell/dell.py
+++ b/llama_stack/distributions/dell/dell.py
@@ -16,6 +16,7 @@ from llama_stack.distributions.template import DistributionTemplate, RunConfigSe
 from llama_stack.providers.inline.inference.sentence_transformers import (
     SentenceTransformersInferenceConfig,
 )
+from llama_stack.providers.remote.vector_io.chroma import ChromaVectorIOConfig
 
 
 def get_distribution_template() -> DistributionTemplate:
@@ -71,9 +72,10 @@ def get_distribution_template() -> DistributionTemplate:
     chromadb_provider = Provider(
         provider_id="chromadb",
         provider_type="remote::chromadb",
-        config={
-            "url": "${env.CHROMA_URL}",
-        },
+        config=ChromaVectorIOConfig.sample_run_config(
+            f"~/.llama/distributions/{name}/",
+            url="${env.CHROMADB_URL:=}",
+        ),
     )
 
     inference_model = ModelInput(
diff --git a/llama_stack/distributions/dell/run-with-safety.yaml b/llama_stack/distributions/dell/run-with-safety.yaml
index ecc6729eb..d89c92aa1 100644
--- a/llama_stack/distributions/dell/run-with-safety.yaml
+++ b/llama_stack/distributions/dell/run-with-safety.yaml
@@ -26,7 +26,10 @@ providers:
   - provider_id: chromadb
     provider_type: remote::chromadb
     config:
-      url: ${env.CHROMA_URL}
+      url: ${env.CHROMADB_URL:=}
+      kvstore:
+        type: sqlite
+        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/dell/}/chroma_remote_registry.db
   safety:
   - provider_id: llama-guard
     provider_type: inline::llama-guard
diff --git a/llama_stack/distributions/dell/run.yaml b/llama_stack/distributions/dell/run.yaml
index fc2553526..7397410ba 100644
--- a/llama_stack/distributions/dell/run.yaml
+++ b/llama_stack/distributions/dell/run.yaml
@@ -22,7 +22,10 @@ providers:
   - provider_id: chromadb
     provider_type: remote::chromadb
     config:
-      url: ${env.CHROMA_URL}
+      url: ${env.CHROMADB_URL:=}
+      kvstore:
+        type: sqlite
+        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/dell/}/chroma_remote_registry.db
   safety:
   - provider_id: llama-guard
     provider_type: inline::llama-guard
diff --git a/llama_stack/providers/remote/inference/tgi/tgi.py b/llama_stack/providers/remote/inference/tgi/tgi.py
index a5bb079ef..323831845 100644
--- a/llama_stack/providers/remote/inference/tgi/tgi.py
+++ b/llama_stack/providers/remote/inference/tgi/tgi.py
@@ -308,9 +308,7 @@ class TGIAdapter(_HfAdapter):
         if not config.url:
             raise ValueError("You must provide a URL in run.yaml (or via the TGI_URL environment variable) to use TGI.")
         log.info(f"Initializing TGI client with url={config.url}")
-        self.client = AsyncInferenceClient(
-            model=config.url,
-        )
+        self.client = AsyncInferenceClient(model=config.url, provider="hf-inference")
         endpoint_info = await self.client.get_endpoint_info()
         self.max_tokens = endpoint_info["max_total_tokens"]
         self.model_id = endpoint_info["model_id"]

From 5bd6cb52fb9dd5a1a4defaeef0fe881cce59efdd Mon Sep 17 00:00:00 2001
From: Krzysztof Malczuk <2000krzysztof@gmail.com>
Date: Wed, 13 Aug 2025 15:14:03 +0100
Subject: [PATCH 12/45] fix: github action canceling valid tasks for checking
 semantic pr title (#3127)

# What does this PR do?
<!-- Provide a short summary of what this PR does and why. Link to
relevant issues if applicable. -->
This PR changes the group name from github.ref to
github.even.pull_request_number. The reason for this is that github.ref
does not act as a unique identifier in the pull_request_target event and
only is unique in pull_request. The github action was getting canceled
was because the group name was not unique in the concurrency section.

<!-- If resolving an issue, uncomment and update the line below -->
Closes #3102

## Test Plan
<!-- Describe the tests you ran to verify your changes with result
summaries. *Provide clear instructions so the plan can be easily
re-executed.* -->
To test this I have created a fake github action and ran it trough act
to see what the github.ref variable produced and what alternatives can
be used. This confirmed that the github.ref was not unique and that
github.event.pull_request_number is unique to the PR.
---
 .github/workflows/semantic-pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/semantic-pr.yml b/.github/workflows/semantic-pr.yml
index 4df7324c4..57a4df646 100644
--- a/.github/workflows/semantic-pr.yml
+++ b/.github/workflows/semantic-pr.yml
@@ -11,7 +11,7 @@ on:
       - synchronize
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 permissions:

From 92aca434a72cc0170c275e857781ebbce1e9e709 Mon Sep 17 00:00:00 2001
From: Francisco Arceo <arceofrancisco@gmail.com>
Date: Wed, 13 Aug 2025 08:46:26 -0600
Subject: [PATCH 13/45] fix: Fix list_sessions() (#3114)

# What does this PR do?
1. Updates `AgentPersistence.list_sessions()` to properly filter out
`Turn` keys from `Session` keys.
2. Adds a suite of unit tests to confirm the `list_sessions()` behavior
and tests the failed sample in
https://github.com/meta-llama/llama-stack/issues/3048

## Fixes https://github.com/meta-llama/llama-stack/issues/3048


## Test Plan
Unit tests added.

---------

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
---
 .../agents/meta_reference/persistence.py      |   6 +-
 .../agent/test_agent_meta_reference.py        | 347 ++++++++++++++++++
 2 files changed, 352 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit/providers/agent/test_agent_meta_reference.py

diff --git a/llama_stack/providers/inline/agents/meta_reference/persistence.py b/llama_stack/providers/inline/agents/meta_reference/persistence.py
index 7a8d99b78..0b234d96c 100644
--- a/llama_stack/providers/inline/agents/meta_reference/persistence.py
+++ b/llama_stack/providers/inline/agents/meta_reference/persistence.py
@@ -191,7 +191,11 @@ class AgentPersistence:
         sessions = []
         for value in values:
             try:
-                session_info = Session(**json.loads(value))
+                data = json.loads(value)
+                if "turn_id" in data:
+                    continue
+
+                session_info = Session(**data)
                 sessions.append(session_info)
             except Exception as e:
                 log.error(f"Error parsing session info: {e}")
diff --git a/tests/unit/providers/agent/test_agent_meta_reference.py b/tests/unit/providers/agent/test_agent_meta_reference.py
new file mode 100644
index 000000000..3fc60024a
--- /dev/null
+++ b/tests/unit/providers/agent/test_agent_meta_reference.py
@@ -0,0 +1,347 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import json
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from llama_stack.apis.agents import Session
+from llama_stack.core.datatypes import User
+from llama_stack.providers.inline.agents.meta_reference.persistence import (
+    AgentPersistence,
+    AgentSessionInfo,
+)
+from llama_stack.providers.utils.kvstore import KVStore
+
+
+@pytest.fixture
+def mock_kvstore():
+    return AsyncMock(spec=KVStore)
+
+
+@pytest.fixture
+def mock_policy():
+    return []
+
+
+@pytest.fixture
+def agent_persistence(mock_kvstore, mock_policy):
+    return AgentPersistence(agent_id="test-agent-123", kvstore=mock_kvstore, policy=mock_policy)
+
+
+@pytest.fixture
+def sample_session():
+    return AgentSessionInfo(
+        session_id="session-123",
+        session_name="Test Session",
+        started_at=datetime.now(UTC),
+        owner=User(principal="user-123", attributes=None),
+        turns=[],
+        identifier="test-session",
+        type="session",
+    )
+
+
+@pytest.fixture
+def sample_session_json(sample_session):
+    return sample_session.model_dump_json()
+
+
+class TestAgentPersistenceListSessions:
+    def setup_mock_kvstore(self, mock_kvstore, session_keys=None, turn_keys=None, invalid_keys=None, custom_data=None):
+        """Helper to setup mock kvstore with sessions, turns, and custom/invalid data
+
+        Args:
+            mock_kvstore: The mock KVStore object
+            session_keys: List of session keys or dict mapping keys to custom session data
+            turn_keys: List of turn keys or dict mapping keys to custom turn data
+            invalid_keys: Dict mapping keys to invalid/corrupt data
+            custom_data: Additional custom data to add to the mock responses
+        """
+        all_keys = []
+        mock_data = {}
+
+        # session keys
+        if session_keys:
+            if isinstance(session_keys, dict):
+                all_keys.extend(session_keys.keys())
+                mock_data.update({k: json.dumps(v) if isinstance(v, dict) else v for k, v in session_keys.items()})
+            else:
+                all_keys.extend(session_keys)
+                for key in session_keys:
+                    session_id = key.split(":")[-1]
+                    mock_data[key] = json.dumps(
+                        {
+                            "session_id": session_id,
+                            "session_name": f"Session {session_id}",
+                            "started_at": datetime.now(UTC).isoformat(),
+                            "turns": [],
+                        }
+                    )
+
+        # turn keys
+        if turn_keys:
+            if isinstance(turn_keys, dict):
+                all_keys.extend(turn_keys.keys())
+                mock_data.update({k: json.dumps(v) if isinstance(v, dict) else v for k, v in turn_keys.items()})
+            else:
+                all_keys.extend(turn_keys)
+                for key in turn_keys:
+                    parts = key.split(":")
+                    session_id = parts[-2]
+                    turn_id = parts[-1]
+                    mock_data[key] = json.dumps(
+                        {
+                            "turn_id": turn_id,
+                            "session_id": session_id,
+                            "input_messages": [],
+                            "started_at": datetime.now(UTC).isoformat(),
+                        }
+                    )
+
+        if invalid_keys:
+            all_keys.extend(invalid_keys.keys())
+            mock_data.update(invalid_keys)
+
+        if custom_data:
+            mock_data.update(custom_data)
+
+        values_list = list(mock_data.values())
+        mock_kvstore.values_in_range.return_value = values_list
+
+        async def mock_get(key):
+            return mock_data.get(key)
+
+        mock_kvstore.get.side_effect = mock_get
+
+        return mock_data
+
+    @pytest.mark.parametrize(
+        "scenario",
+        [
+            {
+                # from this issue: https://github.com/meta-llama/llama-stack/issues/3048
+                "name": "reported_bug",
+                "session_keys": ["session:test-agent-123:1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d"],
+                "turn_keys": [
+                    "session:test-agent-123:1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d:eb7e818f-41fb-49a0-bdd6-464974a2d2ad"
+                ],
+                "expected_sessions": ["1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d"],
+            },
+            {
+                "name": "basic_filtering",
+                "session_keys": ["session:test-agent-123:session-1", "session:test-agent-123:session-2"],
+                "turn_keys": ["session:test-agent-123:session-1:turn-1", "session:test-agent-123:session-1:turn-2"],
+                "expected_sessions": ["session-1", "session-2"],
+            },
+            {
+                "name": "multiple_turns_per_session",
+                "session_keys": ["session:test-agent-123:session-456"],
+                "turn_keys": [
+                    "session:test-agent-123:session-456:turn-789",
+                    "session:test-agent-123:session-456:turn-790",
+                ],
+                "expected_sessions": ["session-456"],
+            },
+            {
+                "name": "multiple_sessions_with_turns",
+                "session_keys": ["session:test-agent-123:session-1", "session:test-agent-123:session-2"],
+                "turn_keys": [
+                    "session:test-agent-123:session-1:turn-1",
+                    "session:test-agent-123:session-1:turn-2",
+                    "session:test-agent-123:session-2:turn-3",
+                ],
+                "expected_sessions": ["session-1", "session-2"],
+            },
+        ],
+    )
+    async def test_list_sessions_key_filtering(self, agent_persistence, mock_kvstore, scenario):
+        self.setup_mock_kvstore(mock_kvstore, session_keys=scenario["session_keys"], turn_keys=scenario["turn_keys"])
+
+        with patch("llama_stack.providers.inline.agents.meta_reference.persistence.log") as mock_log:
+            result = await agent_persistence.list_sessions()
+
+        assert len(result) == len(scenario["expected_sessions"])
+        session_ids = {s.session_id for s in result}
+        for expected_id in scenario["expected_sessions"]:
+            assert expected_id in session_ids
+
+        # no errors should be logged
+        mock_log.error.assert_not_called()
+
+    @pytest.mark.parametrize(
+        "error_scenario",
+        [
+            {
+                "name": "invalid_json",
+                "valid_keys": ["session:test-agent-123:valid-session"],
+                "invalid_data": {"session:test-agent-123:invalid-json": "corrupted-json-data{"},
+                "expected_valid_sessions": ["valid-session"],
+                "expected_error_count": 1,
+            },
+            {
+                "name": "missing_fields",
+                "valid_keys": ["session:test-agent-123:valid-session"],
+                "invalid_data": {
+                    "session:test-agent-123:invalid-schema": json.dumps(
+                        {
+                            "session_id": "invalid-schema",
+                            "session_name": "Missing Fields",
+                            # missing `started_at` and `turns`
+                        }
+                    )
+                },
+                "expected_valid_sessions": ["valid-session"],
+                "expected_error_count": 1,
+            },
+            {
+                "name": "multiple_invalid",
+                "valid_keys": ["session:test-agent-123:valid-session-1", "session:test-agent-123:valid-session-2"],
+                "invalid_data": {
+                    "session:test-agent-123:corrupted-json": "not-valid-json{",
+                    "session:test-agent-123:incomplete-data": json.dumps({"incomplete": "data"}),
+                },
+                "expected_valid_sessions": ["valid-session-1", "valid-session-2"],
+                "expected_error_count": 2,
+            },
+        ],
+    )
+    async def test_list_sessions_error_handling(self, agent_persistence, mock_kvstore, error_scenario):
+        session_keys = {}
+        for key in error_scenario["valid_keys"]:
+            session_id = key.split(":")[-1]
+            session_keys[key] = {
+                "session_id": session_id,
+                "session_name": f"Valid {session_id}",
+                "started_at": datetime.now(UTC).isoformat(),
+                "turns": [],
+            }
+
+        self.setup_mock_kvstore(mock_kvstore, session_keys=session_keys, invalid_keys=error_scenario["invalid_data"])
+
+        with patch("llama_stack.providers.inline.agents.meta_reference.persistence.log") as mock_log:
+            result = await agent_persistence.list_sessions()
+
+        # only valid sessions should be returned
+        assert len(result) == len(error_scenario["expected_valid_sessions"])
+        session_ids = {s.session_id for s in result}
+        for expected_id in error_scenario["expected_valid_sessions"]:
+            assert expected_id in session_ids
+
+        # error should be logged
+        assert mock_log.error.call_count > 0
+        assert mock_log.error.call_count == error_scenario["expected_error_count"]
+
+    async def test_list_sessions_empty(self, agent_persistence, mock_kvstore):
+        mock_kvstore.values_in_range.return_value = []
+
+        result = await agent_persistence.list_sessions()
+
+        assert result == []
+        mock_kvstore.values_in_range.assert_called_once_with(
+            start_key="session:test-agent-123:", end_key="session:test-agent-123:\xff\xff\xff\xff"
+        )
+
+    async def test_list_sessions_properties(self, agent_persistence, mock_kvstore):
+        session_data = {
+            "session_id": "session-123",
+            "session_name": "Test Session",
+            "started_at": datetime.now(UTC).isoformat(),
+            "owner": {"principal": "user-123", "attributes": None},
+            "turns": [],
+        }
+
+        self.setup_mock_kvstore(mock_kvstore, session_keys={"session:test-agent-123:session-123": session_data})
+
+        result = await agent_persistence.list_sessions()
+
+        assert len(result) == 1
+        assert isinstance(result[0], Session)
+        assert result[0].session_id == "session-123"
+        assert result[0].session_name == "Test Session"
+        assert result[0].turns == []
+        assert hasattr(result[0], "started_at")
+
+    async def test_list_sessions_kvstore_exception(self, agent_persistence, mock_kvstore):
+        mock_kvstore.values_in_range.side_effect = Exception("KVStore error")
+
+        with pytest.raises(Exception, match="KVStore error"):
+            await agent_persistence.list_sessions()
+
+    async def test_bug_data_loss_with_real_data(self, agent_persistence, mock_kvstore):
+        # tests the handling of the issue reported in: https://github.com/meta-llama/llama-stack/issues/3048
+        session_data = {
+            "session_id": "1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d",
+            "session_name": "Test Session",
+            "started_at": datetime.now(UTC).isoformat(),
+            "turns": [],
+        }
+
+        turn_data = {
+            "turn_id": "eb7e818f-41fb-49a0-bdd6-464974a2d2ad",
+            "session_id": "1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d",
+            "input_messages": [
+                {"role": "user", "content": "if i had a cluster i would want to call it persistence01", "context": None}
+            ],
+            "steps": [
+                {
+                    "turn_id": "eb7e818f-41fb-49a0-bdd6-464974a2d2ad",
+                    "step_id": "c0f797dd-3d34-4bc5-a8f4-db6af9455132",
+                    "started_at": "2025-08-05T14:31:50.000484Z",
+                    "completed_at": "2025-08-05T14:31:51.303691Z",
+                    "step_type": "inference",
+                    "model_response": {
+                        "role": "assistant",
+                        "content": "OK, I can create a cluster named 'persistence01' for you.",
+                        "stop_reason": "end_of_turn",
+                        "tool_calls": [],
+                    },
+                }
+            ],
+            "output_message": {
+                "role": "assistant",
+                "content": "OK, I can create a cluster named 'persistence01' for you.",
+                "stop_reason": "end_of_turn",
+                "tool_calls": [],
+            },
+            "output_attachments": [],
+            "started_at": "2025-08-05T14:31:49.999950Z",
+            "completed_at": "2025-08-05T14:31:51.305384Z",
+        }
+
+        mock_data = {
+            "session:test-agent-123:1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d": json.dumps(session_data),
+            "session:test-agent-123:1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d:eb7e818f-41fb-49a0-bdd6-464974a2d2ad": json.dumps(
+                turn_data
+            ),
+        }
+
+        mock_kvstore.values_in_range.return_value = list(mock_data.values())
+
+        async def mock_get(key):
+            return mock_data.get(key)
+
+        mock_kvstore.get.side_effect = mock_get
+
+        with patch("llama_stack.providers.inline.agents.meta_reference.persistence.log") as mock_log:
+            result = await agent_persistence.list_sessions()
+
+        assert len(result) == 1
+        assert result[0].session_id == "1f08fd1c-5a9d-459d-a00b-36d4dfa49b7d"
+
+        # confirm no errors logged
+        mock_log.error.assert_not_called()
+
+    async def test_list_sessions_key_range_construction(self, agent_persistence, mock_kvstore):
+        mock_kvstore.values_in_range.return_value = []
+
+        await agent_persistence.list_sessions()
+
+        mock_kvstore.values_in_range.assert_called_once_with(
+            start_key="session:test-agent-123:", end_key="session:test-agent-123:\xff\xff\xff\xff"
+        )

From c9b78602d3a99832dd85197dfb206f1439490945 Mon Sep 17 00:00:00 2001
From: IAN MILLER <75687988+r3v5@users.noreply.github.com>
Date: Wed, 13 Aug 2025 15:56:26 +0100
Subject: [PATCH 14/45] refactor: modify DELETE API endpoints by returning HTTP
 204 No Content + empty body instead of 200 OK + response body with null
 (#3112)

# What does this PR do?
<!-- Provide a short summary of what this PR does and why. Link to
relevant issues if applicable. -->
The purpose of this PR is to make the behavior DELETE API endpoints be
consistent with standard RESTful conventions and eliminate confusion for
API consumers.

Old Behavior
```
HTTP Status: 200 OK
Response Body: null
```

Eg. `curl -X DELETE http://localhost:8321/v1/shields/test-shield`
`null% `
`INFO 2025-08-12 16:11:57,932 console_span_processor:65 telemetry:
15:11:57.929 [INFO] ::1:59805 - "DELETE /v1/shields/test-shield
HTTP/1.1" 200 `

Updated Behavior
```
HTTP Status: 204 No Content
Response Body: empty (no body)
```

Eg.  `curl -X DELETE http://localhost:8321/v1/shields/test-shield`
`INFO 2025-08-12 16:18:16,645 console_span_processor:62 telemetry:
15:18:16.637 [INFO] ::1:60283 - "DELETE /v1/shields/test-shield
HTTP/1.1" 204 `

<!-- If resolving an issue, uncomment and update the line below -->
<!-- Closes #[issue-number] -->
Closes #3090

## Test Plan
<!-- Describe the tests you ran to verify your changes with result
summaries. *Provide clear instructions so the plan can be easily
re-executed.* -->
Run `./scripts/unit-tests.sh`
---
 llama_stack/core/library_client.py | 11 ++++++++++-
 llama_stack/core/server/server.py  |  7 ++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/llama_stack/core/library_client.py b/llama_stack/core/library_client.py
index 5fbbf1aff..a93fe509e 100644
--- a/llama_stack/core/library_client.py
+++ b/llama_stack/core/library_client.py
@@ -380,8 +380,17 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient):
         json_content = json.dumps(convert_pydantic_to_json_value(result))
 
         filtered_body = {k: v for k, v in body.items() if not isinstance(v, LibraryClientUploadFile)}
+
+        status_code = httpx.codes.OK
+
+        if options.method.upper() == "DELETE" and result is None:
+            status_code = httpx.codes.NO_CONTENT
+
+        if status_code == httpx.codes.NO_CONTENT:
+            json_content = ""
+
         mock_response = httpx.Response(
-            status_code=httpx.codes.OK,
+            status_code=status_code,
             content=json_content.encode("utf-8"),
             headers={
                 "Content-Type": "application/json",
diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py
index fe5cc68d7..61ad3e7b3 100644
--- a/llama_stack/core/server/server.py
+++ b/llama_stack/core/server/server.py
@@ -21,10 +21,11 @@ from importlib.metadata import version as parse_version
 from pathlib import Path
 from typing import Annotated, Any, get_origin
 
+import httpx
 import rich.pretty
 import yaml
 from aiohttp import hdrs
-from fastapi import Body, FastAPI, HTTPException, Request
+from fastapi import Body, FastAPI, HTTPException, Request, Response
 from fastapi import Path as FastapiPath
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse, StreamingResponse
@@ -236,6 +237,10 @@ def create_dynamic_typed_route(func: Any, method: str, route: str) -> Callable:
                     result = await maybe_await(value)
                     if isinstance(result, PaginatedResponse) and result.url is None:
                         result.url = route
+
+                    if method.upper() == "DELETE" and result is None:
+                        return Response(status_code=httpx.codes.NO_CONTENT)
+
                     return result
             except Exception as e:
                 if logger.isEnabledFor(logging.DEBUG):

From 0cbd93c5cc44b790c5b08a2f827944c9ac3223d7 Mon Sep 17 00:00:00 2001
From: Kelly Brown <86735520+kelbrown20@users.noreply.github.com>
Date: Wed, 13 Aug 2025 11:06:31 -0400
Subject: [PATCH 15/45] docs: Update blocks formatting in docs/source files
 (#3120)

**Description:**
The standard markdown [!NOTE] format is not supported on Sphinx
generated documentation, replacing those instances. Also updating other
Notes, Tips and Warning blocks throughout the source docs

WIP: Working to update the provider code gen
---
 docs/source/building_applications/responses_vs_agents.md     | 4 +++-
 docs/source/building_applications/tools.md                   | 4 +++-
 docs/source/providers/vector_io/inline_meta-reference.md     | 4 +++-
 docs/source/providers/vector_io/inline_sqlite_vec.md         | 4 +++-
 docs/source/providers/vector_io/remote_milvus.md             | 5 ++++-
 .../source/references/llama_cli_reference/download_models.md | 4 +++-
 docs/source/references/llama_cli_reference/index.md          | 4 +++-
 scripts/provider_codegen.py                                  | 4 ++--
 8 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/docs/source/building_applications/responses_vs_agents.md b/docs/source/building_applications/responses_vs_agents.md
index 3eebfb460..5abe951d6 100644
--- a/docs/source/building_applications/responses_vs_agents.md
+++ b/docs/source/building_applications/responses_vs_agents.md
@@ -2,7 +2,9 @@
 
 Llama Stack (LLS) provides two different APIs for building AI applications with tool calling capabilities: the **Agents API** and the **OpenAI Responses API**. While both enable AI systems to use tools, and maintain full conversation history, they serve different use cases and have distinct characteristics.
 
-> **Note:** For simple and basic inferencing, you may want to use the [Chat Completions API](https://llama-stack.readthedocs.io/en/latest/providers/index.html#chat-completions) directly, before progressing to Agents or Responses API.
+```{note}
+For simple and basic inferencing, you may want to use the [Chat Completions API](https://llama-stack.readthedocs.io/en/latest/providers/index.html#chat-completions) directly, before progressing to Agents or Responses API.
+```
 
 ## Overview
 
diff --git a/docs/source/building_applications/tools.md b/docs/source/building_applications/tools.md
index b19be888c..8a54290ed 100644
--- a/docs/source/building_applications/tools.md
+++ b/docs/source/building_applications/tools.md
@@ -76,7 +76,9 @@ Features:
 - Context retrieval with token limits
 
 
-> **Note:** By default, llama stack run.yaml defines toolgroups for web search, wolfram alpha and rag, that are provided by tavily-search, wolfram-alpha and rag providers.
+```{note}
+By default, llama stack run.yaml defines toolgroups for web search, wolfram alpha and rag, that are provided by tavily-search, wolfram-alpha and rag providers.
+```
 
 ## Model Context Protocol (MCP)
 
diff --git a/docs/source/providers/vector_io/inline_meta-reference.md b/docs/source/providers/vector_io/inline_meta-reference.md
index 0aac445bd..6f269c441 100644
--- a/docs/source/providers/vector_io/inline_meta-reference.md
+++ b/docs/source/providers/vector_io/inline_meta-reference.md
@@ -21,5 +21,7 @@ kvstore:
 
 ## Deprecation Notice
 
-⚠️ **Warning**: Please use the `inline::faiss` provider instead.
+```{warning}
+Please use the `inline::faiss` provider instead.
+```
 
diff --git a/docs/source/providers/vector_io/inline_sqlite_vec.md b/docs/source/providers/vector_io/inline_sqlite_vec.md
index 7ad8eb252..9e5654a50 100644
--- a/docs/source/providers/vector_io/inline_sqlite_vec.md
+++ b/docs/source/providers/vector_io/inline_sqlite_vec.md
@@ -25,5 +25,7 @@ kvstore:
 
 ## Deprecation Notice
 
-⚠️ **Warning**: Please use the `inline::sqlite-vec` provider (notice the hyphen instead of underscore) instead.
+```{warning}
+Please use the `inline::sqlite-vec` provider (notice the hyphen instead of underscore) instead.
+```
 
diff --git a/docs/source/providers/vector_io/remote_milvus.md b/docs/source/providers/vector_io/remote_milvus.md
index 2af64b8bb..075423d04 100644
--- a/docs/source/providers/vector_io/remote_milvus.md
+++ b/docs/source/providers/vector_io/remote_milvus.md
@@ -204,7 +204,10 @@ For more details on TLS configuration, refer to the [TLS setup guide](https://mi
 | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend |
 | `config` | `dict` | No | {} | This configuration allows additional fields to be passed through to the underlying Milvus client. See the [Milvus](https://milvus.io/docs/install-overview.md) documentation for more details about Milvus in general. |
 
-> **Note**: This configuration class accepts additional fields beyond those listed above. You can pass any additional configuration options that will be forwarded to the underlying provider.
+```{note}
+ This configuration class accepts additional fields beyond those listed above. You can pass any additional configuration options that will be forwarded to the underlying provider.
+ ```
+
 
 ## Sample Configuration
 
diff --git a/docs/source/references/llama_cli_reference/download_models.md b/docs/source/references/llama_cli_reference/download_models.md
index e32099023..a9af65349 100644
--- a/docs/source/references/llama_cli_reference/download_models.md
+++ b/docs/source/references/llama_cli_reference/download_models.md
@@ -128,7 +128,9 @@ llama download --source huggingface --model-id Prompt-Guard-86M --ignore-pattern
 
 **Important:** Set your environment variable `HF_TOKEN` or pass in `--hf-token` to the command to validate your access. You can find your token at [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens).
 
-> **Tip:** Default for `llama download` is to run with `--ignore-patterns *.safetensors` since we use the `.pth` files in the `original` folder. For Llama Guard and Prompt Guard, however, we need safetensors. Hence, please run with `--ignore-patterns original` so that safetensors are downloaded and `.pth` files are ignored.
+```{tip}
+Default for `llama download` is to run with `--ignore-patterns *.safetensors` since we use the `.pth` files in the `original` folder. For Llama Guard and Prompt Guard, however, we need safetensors. Hence, please run with `--ignore-patterns original` so that safetensors are downloaded and `.pth` files are ignored.
+```
 
 ## List the downloaded models
 
diff --git a/docs/source/references/llama_cli_reference/index.md b/docs/source/references/llama_cli_reference/index.md
index 4ef76fe7d..09a8b7177 100644
--- a/docs/source/references/llama_cli_reference/index.md
+++ b/docs/source/references/llama_cli_reference/index.md
@@ -152,7 +152,9 @@ llama download --source huggingface --model-id Prompt-Guard-86M --ignore-pattern
 
 **Important:** Set your environment variable `HF_TOKEN` or pass in `--hf-token` to the command to validate your access. You can find your token at [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens).
 
-> **Tip:** Default for `llama download` is to run with `--ignore-patterns *.safetensors` since we use the `.pth` files in the `original` folder. For Llama Guard and Prompt Guard, however, we need safetensors. Hence, please run with `--ignore-patterns original` so that safetensors are downloaded and `.pth` files are ignored.
+```{tip}
+Default for `llama download` is to run with `--ignore-patterns *.safetensors` since we use the `.pth` files in the `original` folder. For Llama Guard and Prompt Guard, however, we need safetensors. Hence, please run with `--ignore-patterns original` so that safetensors are downloaded and `.pth` files are ignored.
+```
 
 ## List the downloaded models
 
diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py
index 84c45fe27..717677c52 100755
--- a/scripts/provider_codegen.py
+++ b/scripts/provider_codegen.py
@@ -187,7 +187,7 @@ def generate_provider_docs(provider_spec: Any, api_name: str) -> str:
 
         if config_info.get("accepts_extra_config"):
             md_lines.append(
-                "> **Note**: This configuration class accepts additional fields beyond those listed above. You can pass any additional configuration options that will be forwarded to the underlying provider."
+                "```{note}\n This configuration class accepts additional fields beyond those listed above. You can pass any additional configuration options that will be forwarded to the underlying provider.\n ```\n"
             )
             md_lines.append("")
 
@@ -232,7 +232,7 @@ def generate_provider_docs(provider_spec: Any, api_name: str) -> str:
     if hasattr(provider_spec, "deprecation_warning") and provider_spec.deprecation_warning:
         md_lines.append("## Deprecation Notice")
         md_lines.append("")
-        md_lines.append(f"⚠️ **Warning**: {provider_spec.deprecation_warning}")
+        md_lines.append(f"```{{warning}}\n{provider_spec.deprecation_warning}\n```")
         md_lines.append("")
 
     if hasattr(provider_spec, "deprecation_error") and provider_spec.deprecation_error:

From 0950168f26ec9489eb7ff6623d270ee903a7a07b Mon Sep 17 00:00:00 2001
From: IAN MILLER <75687988+r3v5@users.noreply.github.com>
Date: Wed, 13 Aug 2025 16:43:41 +0100
Subject: [PATCH 16/45] refactor: replace hardcoded status codes by httpx.codes
 (#3131)

# What does this PR do?
<!-- Provide a short summary of what this PR does and why. Link to
relevant issues if applicable. -->
The purpose of this PR is to eliminate hardcoded status codes in
server's responses and replace it by `httpx.codes` functionality for
better consistency across the whole project and improvement in code
readability.

<!-- If resolving an issue, uncomment and update the line below -->
<!-- Closes #[issue-number] -->

## Test Plan
<!-- Describe the tests you ran to verify your changes with result
summaries. *Provide clear instructions so the plan can be easily
re-executed.* -->
Run `./scripts/unit-tests.sh`
---
 llama_stack/core/server/server.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py
index 61ad3e7b3..d58037f82 100644
--- a/llama_stack/core/server/server.py
+++ b/llama_stack/core/server/server.py
@@ -116,7 +116,7 @@ def translate_exception(exc: Exception) -> HTTPException | RequestValidationErro
 
     if isinstance(exc, RequestValidationError):
         return HTTPException(
-            status_code=400,
+            status_code=httpx.codes.BAD_REQUEST,
             detail={
                 "errors": [
                     {
@@ -129,20 +129,20 @@ def translate_exception(exc: Exception) -> HTTPException | RequestValidationErro
             },
         )
     elif isinstance(exc, ValueError):
-        return HTTPException(status_code=400, detail=f"Invalid value: {str(exc)}")
+        return HTTPException(status_code=httpx.codes.BAD_REQUEST, detail=f"Invalid value: {str(exc)}")
     elif isinstance(exc, BadRequestError):
-        return HTTPException(status_code=400, detail=str(exc))
+        return HTTPException(status_code=httpx.codes.BAD_REQUEST, detail=str(exc))
     elif isinstance(exc, PermissionError | AccessDeniedError):
-        return HTTPException(status_code=403, detail=f"Permission denied: {str(exc)}")
+        return HTTPException(status_code=httpx.codes.FORBIDDEN, detail=f"Permission denied: {str(exc)}")
     elif isinstance(exc, asyncio.TimeoutError | TimeoutError):
-        return HTTPException(status_code=504, detail=f"Operation timed out: {str(exc)}")
+        return HTTPException(status_code=httpx.codes.GATEWAY_TIMEOUT, detail=f"Operation timed out: {str(exc)}")
     elif isinstance(exc, NotImplementedError):
-        return HTTPException(status_code=501, detail=f"Not implemented: {str(exc)}")
+        return HTTPException(status_code=httpx.codes.NOT_IMPLEMENTED, detail=f"Not implemented: {str(exc)}")
     elif isinstance(exc, AuthenticationRequiredError):
-        return HTTPException(status_code=401, detail=f"Authentication required: {str(exc)}")
+        return HTTPException(status_code=httpx.codes.UNAUTHORIZED, detail=f"Authentication required: {str(exc)}")
     else:
         return HTTPException(
-            status_code=500,
+            status_code=httpx.codes.INTERNAL_SERVER_ERROR,
             detail="Internal server error: An unexpected error occurred.",
         )
 
@@ -357,7 +357,7 @@ class ClientVersionMiddleware:
                             await send(
                                 {
                                     "type": "http.response.start",
-                                    "status": 426,
+                                    "status": httpx.codes.UPGRADE_REQUIRED,
                                     "headers": [[b"content-type", b"application/json"]],
                                 }
                             )

From a9081d87b903830b907bea66bcda6b340bf8e212 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Wed, 13 Aug 2025 09:34:56 -0700
Subject: [PATCH 17/45] feat(ci): update Recording workflow trigger and
 concurrency group

---
 .github/workflows/integration-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 9ef49fba3..f330d2c45 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -5,7 +5,7 @@ run-name: Run the integration test suite from tests/integration in replay mode
 on:
   push:
     branches: [ main ]
-  pull_request:
+  pull_request_target:
     branches: [ main ]
     types: [opened, synchronize, reopened]
     paths:
@@ -34,7 +34,7 @@ on:
 
 concurrency:
   # Skip concurrency for pushes to main - each commit should be tested independently
-  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number }}
   cancel-in-progress: true
 
 jobs:

From 25e0553eed5a21773369dfd3d81316db2da39629 Mon Sep 17 00:00:00 2001
From: slekkala1 <swapna942@meta.com>
Date: Wed, 13 Aug 2025 09:47:35 -0700
Subject: [PATCH 18/45] chore: Change moderations api response to Provider
 returned categories (#3098)

# What does this PR do?
To be compliant with model policies for LLAMA, just return the
categories as is from provider, we will lose the OAI compat in
moderations api response.

<!-- If resolving an issue, uncomment and update the line below -->
<!-- Closes #[issue-number] -->

## Test Plan
`SAFETY_MODEL=llama-guard3:8b LLAMA_STACK_CONFIG=starter uv run pytest
-v tests/integration/safety/test_safety.py
--text-model=llama3.2:3b-instruct-fp16
--embedding-model=all-MiniLM-L6-v2 --safety-shield=ollama`
---
 docs/_static/llama-stack-spec.html            |  2 +-
 docs/_static/llama-stack-spec.yaml            |  4 --
 llama_stack/apis/safety/safety.py             | 37 +-------------
 llama_stack/core/routers/safety.py            | 17 +------
 .../inline/safety/llama_guard/llama_guard.py  | 48 ++++---------------
 .../safety/prompt_guard/prompt_guard.py       |  5 +-
 6 files changed, 16 insertions(+), 97 deletions(-)

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index e2c53d4b0..25f916d87 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -16569,7 +16569,7 @@
                         "additionalProperties": {
                             "type": "number"
                         },
-                        "description": "A list of the categories along with their scores as predicted by model. Required set of categories that need to be in response - violence - violence/graphic - harassment - harassment/threatening - hate - hate/threatening - illicit - illicit/violent - sexual - sexual/minors - self-harm - self-harm/intent - self-harm/instructions"
+                        "description": "A list of the categories along with their scores as predicted by model."
                     },
                     "user_message": {
                         "type": "string"
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index 85cec3a78..43e9fa95a 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -12322,10 +12322,6 @@ components:
             type: number
           description: >-
             A list of the categories along with their scores as predicted by model.
-            Required set of categories that need to be in response - violence - violence/graphic
-            - harassment - harassment/threatening - hate - hate/threatening - illicit
-            - illicit/violent - sexual - sexual/minors - self-harm - self-harm/intent
-            - self-harm/instructions
         user_message:
           type: string
         metadata:
diff --git a/llama_stack/apis/safety/safety.py b/llama_stack/apis/safety/safety.py
index 3f374460b..25ee03ec1 100644
--- a/llama_stack/apis/safety/safety.py
+++ b/llama_stack/apis/safety/safety.py
@@ -4,7 +4,7 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from enum import Enum, StrEnum
+from enum import Enum
 from typing import Any, Protocol, runtime_checkable
 
 from pydantic import BaseModel, Field
@@ -15,27 +15,6 @@ from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol
 from llama_stack.schema_utils import json_schema_type, webmethod
 
 
-# OpenAI Categories to return in the response
-class OpenAICategories(StrEnum):
-    """
-    Required set of categories in moderations api response
-    """
-
-    VIOLENCE = "violence"
-    VIOLENCE_GRAPHIC = "violence/graphic"
-    HARRASMENT = "harassment"
-    HARRASMENT_THREATENING = "harassment/threatening"
-    HATE = "hate"
-    HATE_THREATENING = "hate/threatening"
-    ILLICIT = "illicit"
-    ILLICIT_VIOLENT = "illicit/violent"
-    SEXUAL = "sexual"
-    SEXUAL_MINORS = "sexual/minors"
-    SELF_HARM = "self-harm"
-    SELF_HARM_INTENT = "self-harm/intent"
-    SELF_HARM_INSTRUCTIONS = "self-harm/instructions"
-
-
 @json_schema_type
 class ModerationObjectResults(BaseModel):
     """A moderation object.
@@ -43,20 +22,6 @@ class ModerationObjectResults(BaseModel):
     :param categories: A list of the categories, and whether they are flagged or not.
     :param category_applied_input_types: A list of the categories along with the input type(s) that the score applies to.
     :param category_scores: A list of the categories along with their scores as predicted by model.
-    Required set of categories that need to be in response
-    - violence
-    - violence/graphic
-    - harassment
-    - harassment/threatening
-    - hate
-    - hate/threatening
-    - illicit
-    - illicit/violent
-    - sexual
-    - sexual/minors
-    - self-harm
-    - self-harm/intent
-    - self-harm/instructions
     """
 
     flagged: bool
diff --git a/llama_stack/core/routers/safety.py b/llama_stack/core/routers/safety.py
index 9bf2b1bac..c76673d2a 100644
--- a/llama_stack/core/routers/safety.py
+++ b/llama_stack/core/routers/safety.py
@@ -10,7 +10,7 @@ from llama_stack.apis.inference import (
     Message,
 )
 from llama_stack.apis.safety import RunShieldResponse, Safety
-from llama_stack.apis.safety.safety import ModerationObject, OpenAICategories
+from llama_stack.apis.safety.safety import ModerationObject
 from llama_stack.apis.shields import Shield
 from llama_stack.log import get_logger
 from llama_stack.providers.datatypes import RoutingTable
@@ -82,20 +82,5 @@ class SafetyRouter(Safety):
             input=input,
             model=model,
         )
-        self._validate_required_categories_exist(response)
 
         return response
-
-    def _validate_required_categories_exist(self, response: ModerationObject) -> None:
-        """Validate the ProviderImpl response contains the required Open AI moderations categories."""
-        required_categories = list(map(str, OpenAICategories))
-
-        categories = response.results[0].categories
-        category_applied_input_types = response.results[0].category_applied_input_types
-        category_scores = response.results[0].category_scores
-
-        for i in [categories, category_applied_input_types, category_scores]:
-            if not set(required_categories).issubset(set(i.keys())):
-                raise ValueError(
-                    f"ProviderImpl response is missing required categories: {set(required_categories) - set(i.keys())}"
-                )
diff --git a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py
index f83c39a6a..bae744010 100644
--- a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py
+++ b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py
@@ -22,7 +22,7 @@ from llama_stack.apis.safety import (
     SafetyViolation,
     ViolationLevel,
 )
-from llama_stack.apis.safety.safety import ModerationObject, ModerationObjectResults, OpenAICategories
+from llama_stack.apis.safety.safety import ModerationObject, ModerationObjectResults
 from llama_stack.apis.shields import Shield
 from llama_stack.core.datatypes import Api
 from llama_stack.models.llama.datatypes import Role
@@ -72,30 +72,6 @@ SAFETY_CATEGORIES_TO_CODE_MAP = {
 }
 SAFETY_CODE_TO_CATEGORIES_MAP = {v: k for k, v in SAFETY_CATEGORIES_TO_CODE_MAP.items()}
 
-OPENAI_TO_LLAMA_CATEGORIES_MAP = {
-    OpenAICategories.VIOLENCE: [CAT_VIOLENT_CRIMES],
-    OpenAICategories.VIOLENCE_GRAPHIC: [CAT_VIOLENT_CRIMES],
-    OpenAICategories.HARRASMENT: [CAT_CHILD_EXPLOITATION],
-    OpenAICategories.HARRASMENT_THREATENING: [CAT_VIOLENT_CRIMES, CAT_CHILD_EXPLOITATION],
-    OpenAICategories.HATE: [CAT_HATE],
-    OpenAICategories.HATE_THREATENING: [CAT_HATE, CAT_VIOLENT_CRIMES],
-    OpenAICategories.ILLICIT: [CAT_NON_VIOLENT_CRIMES],
-    OpenAICategories.ILLICIT_VIOLENT: [CAT_VIOLENT_CRIMES, CAT_INDISCRIMINATE_WEAPONS],
-    OpenAICategories.SEXUAL: [CAT_SEX_CRIMES, CAT_SEXUAL_CONTENT],
-    OpenAICategories.SEXUAL_MINORS: [CAT_CHILD_EXPLOITATION],
-    OpenAICategories.SELF_HARM: [CAT_SELF_HARM],
-    OpenAICategories.SELF_HARM_INTENT: [CAT_SELF_HARM],
-    OpenAICategories.SELF_HARM_INSTRUCTIONS: [CAT_SELF_HARM, CAT_SPECIALIZED_ADVICE],
-    # These are custom categories that are not in the OpenAI moderation categories
-    "custom/defamation": [CAT_DEFAMATION],
-    "custom/specialized_advice": [CAT_SPECIALIZED_ADVICE],
-    "custom/privacy_violation": [CAT_PRIVACY],
-    "custom/intellectual_property": [CAT_INTELLECTUAL_PROPERTY],
-    "custom/weapons": [CAT_INDISCRIMINATE_WEAPONS],
-    "custom/elections": [CAT_ELECTIONS],
-    "custom/code_interpreter_abuse": [CAT_CODE_INTERPRETER_ABUSE],
-}
-
 
 DEFAULT_LG_V3_SAFETY_CATEGORIES = [
     CAT_VIOLENT_CRIMES,
@@ -424,9 +400,9 @@ class LlamaGuardShield:
             ModerationObject with appropriate configuration
         """
         # Set default values for safe case
-        categories = dict.fromkeys(OPENAI_TO_LLAMA_CATEGORIES_MAP.keys(), False)
-        category_scores = dict.fromkeys(OPENAI_TO_LLAMA_CATEGORIES_MAP.keys(), 1.0)
-        category_applied_input_types = {key: [] for key in OPENAI_TO_LLAMA_CATEGORIES_MAP.keys()}
+        categories = dict.fromkeys(SAFETY_CATEGORIES_TO_CODE_MAP.keys(), False)
+        category_scores = dict.fromkeys(SAFETY_CATEGORIES_TO_CODE_MAP.keys(), 1.0)
+        category_applied_input_types = {key: [] for key in SAFETY_CATEGORIES_TO_CODE_MAP.keys()}
         flagged = False
         user_message = None
         metadata = {}
@@ -453,19 +429,15 @@ class LlamaGuardShield:
                     ],
                 )
 
-            # Get OpenAI categories for the unsafe codes
-            openai_categories = []
-            for code in unsafe_code_list:
-                llama_guard_category = SAFETY_CODE_TO_CATEGORIES_MAP[code]
-                openai_categories.extend(
-                    k for k, v_l in OPENAI_TO_LLAMA_CATEGORIES_MAP.items() if llama_guard_category in v_l
-                )
+            llama_guard_category = [SAFETY_CODE_TO_CATEGORIES_MAP[code] for code in unsafe_code_list]
 
             # Update categories for unsafe content
-            categories = {k: k in openai_categories for k in OPENAI_TO_LLAMA_CATEGORIES_MAP}
-            category_scores = {k: 1.0 if k in openai_categories else 0.0 for k in OPENAI_TO_LLAMA_CATEGORIES_MAP}
+            categories = {k: k in llama_guard_category for k in SAFETY_CATEGORIES_TO_CODE_MAP.keys()}
+            category_scores = {
+                k: 1.0 if k in llama_guard_category else 0.0 for k in SAFETY_CATEGORIES_TO_CODE_MAP.keys()
+            }
             category_applied_input_types = {
-                k: ["text"] if k in openai_categories else [] for k in OPENAI_TO_LLAMA_CATEGORIES_MAP
+                k: ["text"] if k in llama_guard_category else [] for k in SAFETY_CATEGORIES_TO_CODE_MAP.keys()
             }
             flagged = True
             user_message = CANNED_RESPONSE_TEXT
diff --git a/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py b/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py
index 801500dee..c760f0fd1 100644
--- a/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py
+++ b/llama_stack/providers/inline/safety/prompt_guard/prompt_guard.py
@@ -18,6 +18,7 @@ from llama_stack.apis.safety import (
     ShieldStore,
     ViolationLevel,
 )
+from llama_stack.apis.safety.safety import ModerationObject
 from llama_stack.apis.shields import Shield
 from llama_stack.core.utils.model_utils import model_local_dir
 from llama_stack.providers.datatypes import ShieldsProtocolPrivate
@@ -64,8 +65,8 @@ class PromptGuardSafetyImpl(Safety, ShieldsProtocolPrivate):
 
         return await self.shield.run(messages)
 
-    async def run_moderation(self, input: str | list[str], model: str):
-        raise NotImplementedError("run_moderation not implemented for PromptGuard")
+    async def run_moderation(self, input: str | list[str], model: str) -> ModerationObject:
+        raise NotImplementedError("run_moderation is not implemented for Prompt Guard")
 
 
 class PromptGuardShield:

From 2f51273215ded844b62c7532c1e5e53ca46de1d4 Mon Sep 17 00:00:00 2001
From: ehhuang <ehhuang@users.noreply.github.com>
Date: Wed, 13 Aug 2025 09:51:35 -0700
Subject: [PATCH 19/45] fix: huge speed boost (#3132)

# What does this PR do?
make llama stack fast again


## Test Plan
---
 llama_stack/core/server/server.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py
index d58037f82..e9d70fc8d 100644
--- a/llama_stack/core/server/server.py
+++ b/llama_stack/core/server/server.py
@@ -181,7 +181,6 @@ async def sse_generator(event_gen_coroutine):
         event_gen = await event_gen_coroutine
         async for item in event_gen:
             yield create_sse_event(item)
-            await asyncio.sleep(0.01)
     except asyncio.CancelledError:
         logger.info("Generator cancelled")
         if event_gen:

From d6ae54723df7f1b95b896e81ab1f2b66e15e6642 Mon Sep 17 00:00:00 2001
From: ehhuang <ehhuang@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:58:22 -0700
Subject: [PATCH 20/45] chore: setup for performance benchmarking (#3096)

# What does this PR do?
1. Added a simple mock openai-compat server that serves chat/completion
2. Add a benchmark server in EKS that includes mock inference server
3. Add locust (https://locust.io/) file for load testing

## Test Plan
bash apply.sh
kubectl port-forward service/locust-web-ui 8089:8089
Go to localhost:8089 to start a load test

<img width="1392" height="334" alt="image"
src="https://github.com/user-attachments/assets/d6aa3deb-583a-42ed-889b-751262b8e91c"
/>
<img width="1362" height="881" alt="image"
src="https://github.com/user-attachments/assets/6a28b9b4-05e6-44e2-b504-07e60c12d35e"
/>
---
 .../distributions/k8s-benchmark/apply.sh      |  57 +++
 .../k8s-benchmark/locust-k8s.yaml             | 131 +++++++
 .../distributions/k8s-benchmark/locustfile.py |  78 ++++
 .../k8s-benchmark/openai-mock-deployment.yaml |  52 +++
 .../k8s-benchmark/openai-mock-server.py       | 190 ++++++++++
 .../k8s-benchmark/stack-configmap.yaml        | 143 +++++++
 .../k8s-benchmark/stack-k8s.yaml.template     |  87 +++++
 .../k8s-benchmark/stack_run_config.yaml       | 136 +++++++
 .../distributions/k8s/stack-k8s.yaml.template |   6 +-
 pyproject.toml                                |   3 +
 uv.lock                                       | 354 ++++++++++++++++++
 11 files changed, 1234 insertions(+), 3 deletions(-)
 create mode 100755 docs/source/distributions/k8s-benchmark/apply.sh
 create mode 100644 docs/source/distributions/k8s-benchmark/locust-k8s.yaml
 create mode 100644 docs/source/distributions/k8s-benchmark/locustfile.py
 create mode 100644 docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml
 create mode 100644 docs/source/distributions/k8s-benchmark/openai-mock-server.py
 create mode 100644 docs/source/distributions/k8s-benchmark/stack-configmap.yaml
 create mode 100644 docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template
 create mode 100644 docs/source/distributions/k8s-benchmark/stack_run_config.yaml

diff --git a/docs/source/distributions/k8s-benchmark/apply.sh b/docs/source/distributions/k8s-benchmark/apply.sh
new file mode 100755
index 000000000..119a1c849
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/apply.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+# Deploys the benchmark-specific components on top of the base k8s deployment (../k8s/apply.sh).
+
+export MOCK_INFERENCE_PORT=8080
+export STREAM_DELAY_SECONDS=0.005
+
+export POSTGRES_USER=llamastack
+export POSTGRES_DB=llamastack
+export POSTGRES_PASSWORD=llamastack
+
+export INFERENCE_MODEL=meta-llama/Llama-3.2-3B-Instruct
+export SAFETY_MODEL=meta-llama/Llama-Guard-3-1B
+
+export MOCK_INFERENCE_MODEL=mock-inference
+
+# Use llama-stack-benchmark-service as the benchmark server
+export LOCUST_HOST=http://llama-stack-benchmark-service:8323
+export LOCUST_BASE_PATH=/v1/openai/v1
+
+# Use vllm-service as the benchmark server
+# export LOCUST_HOST=http://vllm-server:8000
+# export LOCUST_BASE_PATH=/v1
+
+
+export BENCHMARK_INFERENCE_MODEL=$INFERENCE_MODEL
+
+set -euo pipefail
+set -x
+
+# Deploy benchmark-specific components
+# Deploy OpenAI mock server
+kubectl create configmap openai-mock --from-file=openai-mock-server.py \
+  --dry-run=client -o yaml | kubectl apply --validate=false -f -
+
+envsubst < openai-mock-deployment.yaml | kubectl apply --validate=false -f -
+
+# Create configmap with our custom stack config
+kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \
+  --dry-run=client -o yaml > stack-configmap.yaml
+
+kubectl apply --validate=false -f stack-configmap.yaml
+
+# Deploy our custom llama stack server (overriding the base one)
+envsubst < stack-k8s.yaml.template | kubectl apply --validate=false -f -
+
+# Deploy Locust load testing
+kubectl create configmap locust-script --from-file=locustfile.py \
+  --dry-run=client -o yaml | kubectl apply --validate=false -f -
+
+envsubst < locust-k8s.yaml | kubectl apply --validate=false -f -
diff --git a/docs/source/distributions/k8s-benchmark/locust-k8s.yaml b/docs/source/distributions/k8s-benchmark/locust-k8s.yaml
new file mode 100644
index 000000000..f20a01b2d
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/locust-k8s.yaml
@@ -0,0 +1,131 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: locust-master
+  labels:
+    app: locust
+    role: master
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: locust
+      role: master
+  template:
+    metadata:
+      labels:
+        app: locust
+        role: master
+    spec:
+      containers:
+      - name: locust-master
+        image: locustio/locust:2.31.8
+        ports:
+        - containerPort: 8089  # Web UI
+        - containerPort: 5557  # Master communication
+        env:
+        - name: LOCUST_HOST
+          value: "${LOCUST_HOST}"
+        - name: LOCUST_LOCUSTFILE
+          value: "/locust/locustfile.py"
+        - name: LOCUST_WEB_HOST
+          value: "0.0.0.0"
+        - name: LOCUST_MASTER
+          value: "true"
+        - name: LOCUST_BASE_PATH
+          value: "${LOCUST_BASE_PATH}"
+        - name: INFERENCE_MODEL
+          value: "${BENCHMARK_INFERENCE_MODEL}"
+        volumeMounts:
+        - name: locust-script
+          mountPath: /locust
+        command: ["locust"]
+        args:
+        - "--master"
+        - "--web-host=0.0.0.0"
+        - "--web-port=8089"
+        - "--host=${LOCUST_HOST}"
+        - "--locustfile=/locust/locustfile.py"
+      volumes:
+      - name: locust-script
+        configMap:
+          name: locust-script
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: locust-worker
+  labels:
+    app: locust
+    role: worker
+spec:
+  replicas: 2  # Start with 2 workers, can be scaled up
+  selector:
+    matchLabels:
+      app: locust
+      role: worker
+  template:
+    metadata:
+      labels:
+        app: locust
+        role: worker
+    spec:
+      containers:
+      - name: locust-worker
+        image: locustio/locust:2.31.8
+        env:
+        - name: LOCUST_HOST
+          value: "${LOCUST_HOST}"
+        - name: LOCUST_LOCUSTFILE
+          value: "/locust/locustfile.py"
+        - name: LOCUST_MASTER_HOST
+          value: "locust-master-service"
+        - name: LOCUST_MASTER_PORT
+          value: "5557"
+        - name: INFERENCE_MODEL
+          value: "${BENCHMARK_INFERENCE_MODEL}"
+        - name: LOCUST_BASE_PATH
+          value: "${LOCUST_BASE_PATH}"
+        volumeMounts:
+        - name: locust-script
+          mountPath: /locust
+        command: ["locust"]
+        args:
+        - "--worker"
+        - "--master-host=locust-master-service"
+        - "--master-port=5557"
+        - "--locustfile=/locust/locustfile.py"
+      volumes:
+      - name: locust-script
+        configMap:
+          name: locust-script
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: locust-master-service
+spec:
+  selector:
+    app: locust
+    role: master
+  ports:
+  - name: web-ui
+    port: 8089
+    targetPort: 8089
+  - name: master-comm
+    port: 5557
+    targetPort: 5557
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: locust-web-ui
+spec:
+  selector:
+    app: locust
+    role: master
+  ports:
+  - port: 8089
+    targetPort: 8089
+  type: ClusterIP  # Keep internal, use port-forward to access
diff --git a/docs/source/distributions/k8s-benchmark/locustfile.py b/docs/source/distributions/k8s-benchmark/locustfile.py
new file mode 100644
index 000000000..8e511fa95
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/locustfile.py
@@ -0,0 +1,78 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Locust load testing script for Llama Stack with Prism mock OpenAI provider.
+"""
+
+import random
+from locust import HttpUser, task, between
+import os
+
+base_path = os.getenv("LOCUST_BASE_PATH", "/v1/openai/v1")
+
+MODEL_ID = os.getenv("INFERENCE_MODEL")
+
+class LlamaStackUser(HttpUser):
+    wait_time = between(0.0, 0.0001)
+    
+    def on_start(self):
+        """Setup authentication and test data."""
+        # No auth required for benchmark server
+        self.headers = {
+            "Content-Type": "application/json"
+        }
+        
+        # Test messages of varying lengths
+        self.test_messages = [
+            [{"role": "user", "content": "Hi"}],
+            [{"role": "user", "content": "What is the capital of France?"}],
+            [{"role": "user", "content": "Explain quantum physics in simple terms."}],
+            [{"role": "user", "content": "Write a short story about a robot learning to paint."}],
+            [
+                {"role": "user", "content": "What is machine learning?"},
+                {"role": "assistant", "content": "Machine learning is a subset of AI..."},
+                {"role": "user", "content": "Can you give me a practical example?"}
+            ]
+        ]
+
+    @task(weight=100)
+    def chat_completion_streaming(self):
+        """Test streaming chat completion (20% of requests)."""
+        messages = random.choice(self.test_messages)
+        payload = {
+            "model": MODEL_ID, 
+            "messages": messages,
+            "stream": True,
+            "max_tokens": 100
+        }
+        
+        with self.client.post(
+            f"{base_path}/chat/completions",
+            headers=self.headers,
+            json=payload,
+            stream=True,
+            catch_response=True
+        ) as response:
+            if response.status_code == 200:
+                chunks_received = 0
+                try:
+                    for line in response.iter_lines():
+                        if line:
+                            line_str = line.decode('utf-8')
+                            if line_str.startswith('data: '):
+                                chunks_received += 1
+                                if line_str.strip() == 'data: [DONE]':
+                                    break
+                    
+                    if chunks_received > 0:
+                        response.success()
+                    else:
+                        response.failure("No streaming chunks received")
+                except Exception as e:
+                    response.failure(f"Streaming error: {e}")
+            else:
+                response.failure(f"HTTP {response.status_code}: {response.text}")
diff --git a/docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml b/docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml
new file mode 100644
index 000000000..c72921281
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml
@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: openai-mock
+  labels:
+    app: openai-mock
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: openai-mock
+  template:
+    metadata:
+      labels:
+        app: openai-mock
+    spec:
+      containers:
+      - name: openai-mock
+        image: python:3.12-slim
+        ports:
+        - containerPort: ${MOCK_INFERENCE_PORT}
+        env:
+        - name: PORT
+          value: "${MOCK_INFERENCE_PORT}"
+        - name: MOCK_MODELS
+          value: "${MOCK_INFERENCE_MODEL}"
+        - name: STREAM_DELAY_SECONDS
+          value: "${STREAM_DELAY_SECONDS}"
+        command: ["sh", "-c"]
+        args:
+        - |
+          pip install flask &&
+          python /app/openai-mock-server.py --port ${MOCK_INFERENCE_PORT}
+        volumeMounts:
+        - name: openai-mock-script
+          mountPath: /app
+      volumes:
+      - name: openai-mock-script
+        configMap:
+          name: openai-mock
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openai-mock-service
+spec:
+  selector:
+    app: openai-mock
+  ports:
+  - port: 8080
+    targetPort: 8080
+  type: ClusterIP
diff --git a/docs/source/distributions/k8s-benchmark/openai-mock-server.py b/docs/source/distributions/k8s-benchmark/openai-mock-server.py
new file mode 100644
index 000000000..46c923b60
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/openai-mock-server.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+OpenAI-compatible mock server that returns:
+- Hardcoded /models response for consistent validation
+- Valid OpenAI-formatted chat completion responses with dynamic content
+"""
+
+from flask import Flask, request, jsonify, Response
+import time
+import random
+import uuid
+import json
+import argparse
+import os
+
+app = Flask(__name__)
+
+# Models from environment variables
+def get_models():
+    models_str = os.getenv("MOCK_MODELS", "mock-inference")
+    model_ids = [m.strip() for m in models_str.split(",") if m.strip()]
+    
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": model_id,
+                "object": "model",
+                "created": 1234567890,
+                "owned_by": "vllm"
+            }
+            for model_id in model_ids
+        ]
+    }
+
+def generate_random_text(length=50):
+    """Generate random but coherent text for responses."""
+    words = [
+        "Hello", "there", "I'm", "an", "AI", "assistant", "ready", "to", "help", "you",
+        "with", "your", "questions", "and", "tasks", "today", "Let", "me","know", "what",
+        "you'd", "like", "to", "discuss", "or", "explore", "together", "I", "can", "assist",
+        "with", "various", "topics", "including", "coding", "writing", "analysis", "and", "more"
+    ]
+    return " ".join(random.choices(words, k=length))
+
+@app.route('/models', methods=['GET'])
+def list_models():
+    models = get_models()
+    print(f"[MOCK] Returning models: {[m['id'] for m in models['data']]}")
+    return jsonify(models)
+
+@app.route('/chat/completions', methods=['POST'])
+def chat_completions():
+    """Return OpenAI-formatted chat completion responses."""
+    data = request.get_json()
+    default_model = get_models()['data'][0]['id']
+    model = data.get('model', default_model)
+    messages = data.get('messages', [])
+    stream = data.get('stream', False)
+     
+    print(f"[MOCK] Chat completion request - model: {model}, stream: {stream}")
+    
+    if stream:
+        return handle_streaming_completion(model, messages)
+    else:
+        return handle_non_streaming_completion(model, messages)
+
+def handle_non_streaming_completion(model, messages):
+    response_text = generate_random_text(random.randint(20, 80))
+    
+    # Calculate realistic token counts
+    prompt_tokens = sum(len(str(msg.get('content', '')).split()) for msg in messages)
+    completion_tokens = len(response_text.split())
+    
+    response = {
+        "id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": response_text
+                },
+                "finish_reason": "stop"
+            }
+        ],
+        "usage": {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens
+        }
+    }
+    
+    return jsonify(response)
+
+def handle_streaming_completion(model, messages):
+    def generate_stream():
+        # Generate response text
+        full_response = generate_random_text(random.randint(30, 100))
+        words = full_response.split()
+        
+        # Send initial chunk
+        initial_chunk = {
+            "id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
+            "object": "chat.completion.chunk",
+            "created": int(time.time()),
+            "model": model,
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"role": "assistant", "content": ""}
+                }
+            ]
+        }
+        yield f"data: {json.dumps(initial_chunk)}\n\n"
+        
+        # Send word by word
+        for i, word in enumerate(words):
+            chunk = {
+                "id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
+                "object": "chat.completion.chunk", 
+                "created": int(time.time()),
+                "model": model,
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {"content": f"{word} " if i < len(words) - 1 else word}
+                    }
+                ]
+            }
+            yield f"data: {json.dumps(chunk)}\n\n"
+            # Configurable delay to simulate realistic streaming
+            stream_delay = float(os.getenv("STREAM_DELAY_SECONDS", "0.005"))
+            time.sleep(stream_delay)
+        
+        # Send final chunk
+        final_chunk = {
+            "id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
+            "object": "chat.completion.chunk",
+            "created": int(time.time()),
+            "model": model,
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"content": ""},
+                    "finish_reason": "stop"
+                }
+            ]
+        }
+        yield f"data: {json.dumps(final_chunk)}\n\n"
+        yield "data: [DONE]\n\n"
+    
+    return Response(
+        generate_stream(),
+        mimetype='text/event-stream',
+        headers={
+            'Cache-Control': 'no-cache',
+            'Connection': 'keep-alive',
+            'Access-Control-Allow-Origin': '*',
+        }
+    )
+
+@app.route('/health', methods=['GET'])
+def health():
+    return jsonify({"status": "healthy", "type": "openai-mock"})
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='OpenAI-compatible mock server')
+    parser.add_argument('--port', type=int, default=8081, 
+                       help='Port to run the server on (default: 8081)')
+    args = parser.parse_args()
+    
+    port = args.port
+    
+    models = get_models()
+    print("Starting OpenAI-compatible mock server...")
+    print(f"- /models endpoint with: {[m['id'] for m in models['data']]}")
+    print("- OpenAI-formatted chat/completion responses with dynamic content")
+    print("- Streaming support with valid SSE format")
+    print(f"- Listening on: http://0.0.0.0:{port}")
+    app.run(host='0.0.0.0', port=port, debug=False)
diff --git a/docs/source/distributions/k8s-benchmark/stack-configmap.yaml b/docs/source/distributions/k8s-benchmark/stack-configmap.yaml
new file mode 100644
index 000000000..653e66756
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/stack-configmap.yaml
@@ -0,0 +1,143 @@
+apiVersion: v1
+data:
+  stack_run_config.yaml: |
+    version: '2'
+    image_name: kubernetes-benchmark-demo
+    apis:
+    - agents
+    - inference
+    - safety
+    - telemetry
+    - tool_runtime
+    - vector_io
+    providers:
+      inference:
+      - provider_id: vllm-inference
+        provider_type: remote::vllm
+        config:
+          url: ${env.VLLM_URL:=http://localhost:8000/v1}
+          max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+          api_token: ${env.VLLM_API_TOKEN:=fake}
+          tls_verify: ${env.VLLM_TLS_VERIFY:=true}
+      - provider_id: vllm-safety
+        provider_type: remote::vllm
+        config:
+          url: ${env.VLLM_SAFETY_URL:=http://localhost:8000/v1}
+          max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+          api_token: ${env.VLLM_API_TOKEN:=fake}
+          tls_verify: ${env.VLLM_TLS_VERIFY:=true}
+      - provider_id: mock-vllm-inference
+        provider_type: remote::vllm
+        config:
+          url: http://openai-mock-service:${env.MOCK_INFERENCE_PORT}
+          max_tokens: 4096
+          api_token: fake
+          tls_verify: false
+      - provider_id: sentence-transformers
+        provider_type: inline::sentence-transformers
+        config: {}
+      vector_io:
+      - provider_id: ${env.ENABLE_CHROMADB:+chromadb}
+        provider_type: remote::chromadb
+        config:
+          url: ${env.CHROMADB_URL:=}
+          kvstore:
+            type: postgres
+            host: ${env.POSTGRES_HOST:=localhost}
+            port: ${env.POSTGRES_PORT:=5432}
+            db: ${env.POSTGRES_DB:=llamastack}
+            user: ${env.POSTGRES_USER:=llamastack}
+            password: ${env.POSTGRES_PASSWORD:=llamastack}
+      safety:
+      - provider_id: llama-guard
+        provider_type: inline::llama-guard
+        config:
+          excluded_categories: []
+      agents:
+      - provider_id: meta-reference
+        provider_type: inline::meta-reference
+        config:
+          persistence_store:
+            type: postgres
+            host: ${env.POSTGRES_HOST:=localhost}
+            port: ${env.POSTGRES_PORT:=5432}
+            db: ${env.POSTGRES_DB:=llamastack}
+            user: ${env.POSTGRES_USER:=llamastack}
+            password: ${env.POSTGRES_PASSWORD:=llamastack}
+          responses_store:
+            type: postgres
+            host: ${env.POSTGRES_HOST:=localhost}
+            port: ${env.POSTGRES_PORT:=5432}
+            db: ${env.POSTGRES_DB:=llamastack}
+            user: ${env.POSTGRES_USER:=llamastack}
+            password: ${env.POSTGRES_PASSWORD:=llamastack}
+      telemetry:
+      - provider_id: meta-reference
+        provider_type: inline::meta-reference
+        config:
+          service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
+          sinks: ${env.TELEMETRY_SINKS:=console}
+      tool_runtime:
+      - provider_id: brave-search
+        provider_type: remote::brave-search
+        config:
+          api_key: ${env.BRAVE_SEARCH_API_KEY:+}
+          max_results: 3
+      - provider_id: tavily-search
+        provider_type: remote::tavily-search
+        config:
+          api_key: ${env.TAVILY_SEARCH_API_KEY:+}
+          max_results: 3
+      - provider_id: rag-runtime
+        provider_type: inline::rag-runtime
+        config: {}
+      - provider_id: model-context-protocol
+        provider_type: remote::model-context-protocol
+        config: {}
+    metadata_store:
+      type: postgres
+      host: ${env.POSTGRES_HOST:=localhost}
+      port: ${env.POSTGRES_PORT:=5432}
+      db: ${env.POSTGRES_DB:=llamastack}
+      user: ${env.POSTGRES_USER:=llamastack}
+      password: ${env.POSTGRES_PASSWORD:=llamastack}
+      table_name: llamastack_kvstore
+    inference_store:
+      type: postgres
+      host: ${env.POSTGRES_HOST:=localhost}
+      port: ${env.POSTGRES_PORT:=5432}
+      db: ${env.POSTGRES_DB:=llamastack}
+      user: ${env.POSTGRES_USER:=llamastack}
+      password: ${env.POSTGRES_PASSWORD:=llamastack}
+    models:
+    - metadata:
+        embedding_dimension: 384
+      model_id: all-MiniLM-L6-v2
+      provider_id: sentence-transformers
+      model_type: embedding
+    - model_id: ${env.INFERENCE_MODEL}
+      provider_id: vllm-inference
+      model_type: llm
+    - model_id: ${env.SAFETY_MODEL}
+      provider_id: vllm-safety
+      model_type: llm
+    - model_id: ${env.MOCK_INFERENCE_MODEL}
+      provider_id: mock-vllm-inference
+      model_type: llm
+    shields:
+    - shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
+    vector_dbs: []
+    datasets: []
+    scoring_fns: []
+    benchmarks: []
+    tool_groups:
+    - toolgroup_id: builtin::websearch
+      provider_id: tavily-search
+    - toolgroup_id: builtin::rag
+      provider_id: rag-runtime
+    server:
+      port: 8323
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: llama-stack-config
diff --git a/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template b/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template
new file mode 100644
index 000000000..bc14d5124
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template
@@ -0,0 +1,87 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: llama-benchmark-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llama-stack-benchmark-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: llama-stack-benchmark
+      app.kubernetes.io/component: server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: llama-stack-benchmark
+        app.kubernetes.io/component: server
+    spec:
+      containers:
+      - name: llama-stack-benchmark
+        image: llamastack/distribution-starter:latest
+        imagePullPolicy: Always # since we have specified latest instead of a version
+        env:
+        - name: ENABLE_CHROMADB
+          value: "true"
+        - name: CHROMADB_URL
+          value: http://chromadb.default.svc.cluster.local:6000
+        - name: POSTGRES_HOST
+          value: postgres-server.default.svc.cluster.local
+        - name: POSTGRES_PORT
+          value: "5432"
+        - name: INFERENCE_MODEL
+          value: "${INFERENCE_MODEL}"
+        - name: SAFETY_MODEL
+          value: "${SAFETY_MODEL}"
+        - name: TAVILY_SEARCH_API_KEY
+          value: "${TAVILY_SEARCH_API_KEY}"
+        - name: MOCK_INFERENCE_PORT
+          value: "${MOCK_INFERENCE_PORT}"
+        - name: VLLM_URL
+          value: http://vllm-server.default.svc.cluster.local:8000/v1
+        - name: VLLM_MAX_TOKENS
+          value: "3072"
+        - name: VLLM_SAFETY_URL
+          value: http://vllm-server-safety.default.svc.cluster.local:8001/v1
+        - name: VLLM_TLS_VERIFY
+          value: "false"
+        - name: MOCK_INFERENCE_MODEL
+          value: "${MOCK_INFERENCE_MODEL}"
+        command: ["python", "-m", "llama_stack.core.server.server", "/etc/config/stack_run_config.yaml", "--port", "8323"]
+        ports:
+          - containerPort: 8323
+        volumeMounts:
+          - name: llama-storage
+            mountPath: /root/.llama
+          - name: llama-config
+            mountPath: /etc/config
+      volumes:
+      - name: llama-storage
+        persistentVolumeClaim:
+          claimName: llama-benchmark-pvc
+      - name: llama-config
+        configMap:
+          name: llama-stack-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: llama-stack-benchmark-service
+spec:
+  selector:
+    app.kubernetes.io/name: llama-stack-benchmark
+    app.kubernetes.io/component: server
+  ports:
+  - name: http
+    port: 8323
+    targetPort: 8323
+  type: ClusterIP
diff --git a/docs/source/distributions/k8s-benchmark/stack_run_config.yaml b/docs/source/distributions/k8s-benchmark/stack_run_config.yaml
new file mode 100644
index 000000000..ad56be047
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/stack_run_config.yaml
@@ -0,0 +1,136 @@
+version: '2'
+image_name: kubernetes-benchmark-demo
+apis:
+- agents
+- inference
+- safety
+- telemetry
+- tool_runtime
+- vector_io
+providers:
+  inference:
+  - provider_id: vllm-inference
+    provider_type: remote::vllm
+    config:
+      url: ${env.VLLM_URL:=http://localhost:8000/v1}
+      max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+      api_token: ${env.VLLM_API_TOKEN:=fake}
+      tls_verify: ${env.VLLM_TLS_VERIFY:=true}
+  - provider_id: vllm-safety
+    provider_type: remote::vllm
+    config:
+      url: ${env.VLLM_SAFETY_URL:=http://localhost:8000/v1}
+      max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+      api_token: ${env.VLLM_API_TOKEN:=fake}
+      tls_verify: ${env.VLLM_TLS_VERIFY:=true}
+  - provider_id: mock-vllm-inference
+    provider_type: remote::vllm
+    config:
+      url: http://openai-mock-service:${env.MOCK_INFERENCE_PORT}
+      max_tokens: 4096
+      api_token: fake
+      tls_verify: false
+  - provider_id: sentence-transformers
+    provider_type: inline::sentence-transformers
+    config: {}
+  vector_io:
+  - provider_id: ${env.ENABLE_CHROMADB:+chromadb}
+    provider_type: remote::chromadb
+    config:
+      url: ${env.CHROMADB_URL:=}
+      kvstore:
+        type: postgres
+        host: ${env.POSTGRES_HOST:=localhost}
+        port: ${env.POSTGRES_PORT:=5432}
+        db: ${env.POSTGRES_DB:=llamastack}
+        user: ${env.POSTGRES_USER:=llamastack}
+        password: ${env.POSTGRES_PASSWORD:=llamastack}
+  safety:
+  - provider_id: llama-guard
+    provider_type: inline::llama-guard
+    config:
+      excluded_categories: []
+  agents:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config:
+      persistence_store:
+        type: postgres
+        host: ${env.POSTGRES_HOST:=localhost}
+        port: ${env.POSTGRES_PORT:=5432}
+        db: ${env.POSTGRES_DB:=llamastack}
+        user: ${env.POSTGRES_USER:=llamastack}
+        password: ${env.POSTGRES_PASSWORD:=llamastack}
+      responses_store:
+        type: postgres
+        host: ${env.POSTGRES_HOST:=localhost}
+        port: ${env.POSTGRES_PORT:=5432}
+        db: ${env.POSTGRES_DB:=llamastack}
+        user: ${env.POSTGRES_USER:=llamastack}
+        password: ${env.POSTGRES_PASSWORD:=llamastack}
+  telemetry:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config:
+      service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
+      sinks: ${env.TELEMETRY_SINKS:=console}
+  tool_runtime:
+  - provider_id: brave-search
+    provider_type: remote::brave-search
+    config:
+      api_key: ${env.BRAVE_SEARCH_API_KEY:+}
+      max_results: 3
+  - provider_id: tavily-search
+    provider_type: remote::tavily-search
+    config:
+      api_key: ${env.TAVILY_SEARCH_API_KEY:+}
+      max_results: 3
+  - provider_id: rag-runtime
+    provider_type: inline::rag-runtime
+    config: {}
+  - provider_id: model-context-protocol
+    provider_type: remote::model-context-protocol
+    config: {}
+metadata_store:
+  type: postgres
+  host: ${env.POSTGRES_HOST:=localhost}
+  port: ${env.POSTGRES_PORT:=5432}
+  db: ${env.POSTGRES_DB:=llamastack}
+  user: ${env.POSTGRES_USER:=llamastack}
+  password: ${env.POSTGRES_PASSWORD:=llamastack}
+  table_name: llamastack_kvstore
+inference_store:
+  type: postgres
+  host: ${env.POSTGRES_HOST:=localhost}
+  port: ${env.POSTGRES_PORT:=5432}
+  db: ${env.POSTGRES_DB:=llamastack}
+  user: ${env.POSTGRES_USER:=llamastack}
+  password: ${env.POSTGRES_PASSWORD:=llamastack}
+models:
+- metadata:
+    embedding_dimension: 384
+  model_id: all-MiniLM-L6-v2
+  provider_id: sentence-transformers
+  model_type: embedding
+- model_id: ${env.INFERENCE_MODEL}
+  provider_id: vllm-inference
+  model_type: llm
+- model_id: ${env.SAFETY_MODEL}
+  provider_id: vllm-safety
+  model_type: llm
+- model_id: ${env.MOCK_INFERENCE_MODEL}
+  provider_id: mock-vllm-inference
+  model_type: llm
+shields:
+- shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
+vector_dbs: []
+datasets: []
+scoring_fns: []
+benchmarks: []
+tool_groups:
+- toolgroup_id: builtin::websearch
+  provider_id: tavily-search
+- toolgroup_id: builtin::rag
+  provider_id: rag-runtime
+server:
+  port: 8323
diff --git a/docs/source/distributions/k8s/stack-k8s.yaml.template b/docs/source/distributions/k8s/stack-k8s.yaml.template
index ad5d2c716..dfc049f4f 100644
--- a/docs/source/distributions/k8s/stack-k8s.yaml.template
+++ b/docs/source/distributions/k8s/stack-k8s.yaml.template
@@ -40,19 +40,19 @@ spec:
           value: "3072"
         - name: VLLM_SAFETY_URL
           value: http://vllm-server-safety.default.svc.cluster.local:8001/v1
+        - name: VLLM_TLS_VERIFY
+          value: "false"
         - name: POSTGRES_HOST
           value: postgres-server.default.svc.cluster.local
         - name: POSTGRES_PORT
           value: "5432"
-        - name: VLLM_TLS_VERIFY
-          value: "false"
         - name: INFERENCE_MODEL
           value: "${INFERENCE_MODEL}"
         - name: SAFETY_MODEL
           value: "${SAFETY_MODEL}"
         - name: TAVILY_SEARCH_API_KEY
           value: "${TAVILY_SEARCH_API_KEY}"
-        command: ["python", "-m", "llama_stack.core.server.server", "--config", "/etc/config/stack_run_config.yaml", "--port", "8321"]
+        command: ["python", "-m", "llama_stack.core.server.server", "/etc/config/stack_run_config.yaml", "--port", "8321"]
         ports:
           - containerPort: 8321
         volumeMounts:
diff --git a/pyproject.toml b/pyproject.toml
index 1b0850631..db0ad1f00 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -140,6 +140,9 @@ docs = [
     "requests",
 ]
 codegen = ["rich", "pydantic", "jinja2>=3.1.6"]
+benchmark = [
+    "locust>=2.37.14",
+]
 
 [project.urls]
 Homepage = "https://github.com/meta-llama/llama-stack"
diff --git a/uv.lock b/uv.lock
index 9f4ba4adb..4c56816ef 100644
--- a/uv.lock
+++ b/uv.lock
@@ -290,6 +290,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a9/cf/45fb5261ece3e6b9817d3d82b2f343a505fd58674a92577923bc500bd1aa/bcrypt-4.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:e53e074b120f2877a35cc6c736b8eb161377caae8925c17688bd46ba56daaa5b", size = 152799, upload-time = "2025-02-28T01:23:53.139Z" },
 ]
 
+[[package]]
+name = "bidict"
+version = "0.23.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" },
+]
+
 [[package]]
 name = "black"
 version = "25.1.0"
@@ -347,6 +356,44 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9e/f6/776782c20b71b4da290ed0b25ccec0cbfca924d20f2ec26078876bce6d29/braintrust_core-0.0.59-py3-none-any.whl", hash = "sha256:b9be128e1c1b4c376f082e81d314c1938aa9b8c0398ab11df4ad29fad8e655c1", size = 4441, upload-time = "2025-05-12T22:05:12.088Z" },
 ]
 
+[[package]]
+name = "brotli"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270, upload-time = "2023-09-07T14:05:41.643Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693, upload-time = "2024-10-18T12:32:23.824Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489, upload-time = "2024-10-18T12:32:25.641Z" },
+    { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081, upload-time = "2023-09-07T14:03:57.967Z" },
+    { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244, upload-time = "2023-09-07T14:03:59.319Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505, upload-time = "2023-09-07T14:04:01.327Z" },
+    { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152, upload-time = "2023-09-07T14:04:03.033Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252, upload-time = "2023-09-07T14:04:04.675Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955, upload-time = "2023-09-07T14:04:06.585Z" },
+    { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304, upload-time = "2023-09-07T14:04:08.668Z" },
+    { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452, upload-time = "2023-09-07T14:04:10.736Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751, upload-time = "2023-09-07T14:04:12.875Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757, upload-time = "2023-09-07T14:04:14.551Z" },
+    { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146, upload-time = "2024-10-18T12:32:27.257Z" },
+    { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055, upload-time = "2024-10-18T12:32:29.376Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102, upload-time = "2024-10-18T12:32:31.371Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029, upload-time = "2024-10-18T12:32:33.293Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276, upload-time = "2023-09-07T14:04:16.49Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255, upload-time = "2023-09-07T14:04:17.83Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681, upload-time = "2024-10-18T12:32:34.942Z" },
+    { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475, upload-time = "2024-10-18T12:32:36.485Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173, upload-time = "2024-10-18T12:32:37.978Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803, upload-time = "2024-10-18T12:32:39.606Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946, upload-time = "2024-10-18T12:32:41.679Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707, upload-time = "2024-10-18T12:32:43.478Z" },
+    { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231, upload-time = "2024-10-18T12:32:45.224Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157, upload-time = "2024-10-18T12:32:46.894Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122, upload-time = "2024-10-18T12:32:48.844Z" },
+    { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206, upload-time = "2024-10-18T12:32:51.198Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804, upload-time = "2024-10-18T12:32:52.661Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517, upload-time = "2024-10-18T12:32:54.066Z" },
+]
+
 [[package]]
 name = "build"
 version = "1.2.2.post1"
@@ -558,6 +605,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" },
 ]
 
+[[package]]
+name = "configargparse"
+version = "1.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/85/4d/6c9ef746dfcc2a32e26f3860bb4a011c008c392b83eabdfb598d1a8bbe5d/configargparse-1.7.1.tar.gz", hash = "sha256:79c2ddae836a1e5914b71d58e4b9adbd9f7779d4e6351a637b7d2d9b6c46d3d9", size = 43958, upload-time = "2025-05-23T14:26:17.369Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/31/28/d28211d29bcc3620b1fece85a65ce5bb22f18670a03cd28ea4b75ede270c/configargparse-1.7.1-py3-none-any.whl", hash = "sha256:8b586a31f9d873abd1ca527ffbe58863c99f36d896e2829779803125e83be4b6", size = 25607, upload-time = "2025-05-23T14:26:15.923Z" },
+]
+
 [[package]]
 name = "coverage"
 version = "7.10.1"
@@ -872,6 +928,49 @@ dependencies = [
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/6b/b6/82c7e601d6d3c3278c40b7bd35e17e82aa227f050aa9f66cb7b7fce29471/fire-0.7.0.tar.gz", hash = "sha256:961550f07936eaf65ad1dc8360f2b2bf8408fad46abbfa4d2a3794f8d2a95cdf", size = 87189, upload-time = "2024-10-01T14:29:31.585Z" }
 
+[[package]]
+name = "flask"
+version = "3.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "blinker" },
+    { name = "click" },
+    { name = "itsdangerous" },
+    { name = "jinja2" },
+    { name = "markupsafe" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/de/e47735752347f4128bcf354e0da07ef311a78244eba9e3dc1d4a5ab21a98/flask-3.1.1.tar.gz", hash = "sha256:284c7b8f2f58cb737f0cf1c30fd7eaf0ccfcde196099d24ecede3fc2005aa59e", size = 753440, upload-time = "2025-05-13T15:01:17.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3d/68/9d4508e893976286d2ead7f8f571314af6c2037af34853a30fd769c02e9d/flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c", size = 103305, upload-time = "2025-05-13T15:01:15.591Z" },
+]
+
+[[package]]
+name = "flask-cors"
+version = "6.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "flask" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/37/bcfa6c7d5eec777c4c7cf45ce6b27631cebe5230caf88d85eadd63edd37a/flask_cors-6.0.1.tar.gz", hash = "sha256:d81bcb31f07b0985be7f48406247e9243aced229b7747219160a0559edd678db", size = 13463, upload-time = "2025-06-11T01:32:08.518Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/17/f8/01bf35a3afd734345528f98d0353f2a978a476528ad4d7e78b70c4d149dd/flask_cors-6.0.1-py3-none-any.whl", hash = "sha256:c7b2cbfb1a31aa0d2e5341eea03a6805349f7a61647daee1a15c46bbe981494c", size = 13244, upload-time = "2025-06-11T01:32:07.352Z" },
+]
+
+[[package]]
+name = "flask-login"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "flask" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c3/6e/2f4e13e373bb49e68c02c51ceadd22d172715a06716f9299d9df01b6ddb2/Flask-Login-0.6.3.tar.gz", hash = "sha256:5e23d14a607ef12806c699590b89d0f0e0d67baeec599d75947bf9c147330333", size = 48834, upload-time = "2023-10-30T14:53:21.151Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/59/f5/67e9cc5c2036f58115f9fe0f00d203cf6780c3ff8ae0e705e7a9d9e8ff9e/Flask_Login-0.6.3-py3-none-any.whl", hash = "sha256:849b25b82a436bf830a054e74214074af59097171562ab10bfa999e6b78aae5d", size = 17303, upload-time = "2023-10-30T14:53:19.636Z" },
+]
+
 [[package]]
 name = "flatbuffers"
 version = "25.2.10"
@@ -955,6 +1054,77 @@ http = [
     { name = "aiohttp" },
 ]
 
+[[package]]
+name = "gevent"
+version = "25.5.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" },
+    { name = "greenlet", marker = "platform_python_implementation == 'CPython'" },
+    { name = "zope-event" },
+    { name = "zope-interface" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f1/58/267e8160aea00ab00acd2de97197eecfe307064a376fb5c892870a8a6159/gevent-25.5.1.tar.gz", hash = "sha256:582c948fa9a23188b890d0bc130734a506d039a2e5ad87dae276a456cc683e61", size = 6388207, upload-time = "2025-05-12T12:57:59.833Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/58/c5/cf71423666a0b83db3d7e3f85788bc47d573fca5fe62b798fe2c4273de7c/gevent-25.5.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:d87c0a1bd809d8f70f96b9b229779ec6647339830b8888a192beed33ac8d129f", size = 2909333, upload-time = "2025-05-12T11:11:34.883Z" },
+    { url = "https://files.pythonhosted.org/packages/26/7e/d2f174ee8bec6eb85d961ca203bc599d059c857b8412e367b8fa206603a5/gevent-25.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b87a4b66edb3808d4d07bbdb0deed5a710cf3d3c531e082759afd283758bb649", size = 1788420, upload-time = "2025-05-12T11:52:30.306Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/f3/3aba8c147b9108e62ba348c726fe38ae69735a233db425565227336e8ce6/gevent-25.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f076779050029a82feb0cb1462021d3404d22f80fa76a181b1a7889cd4d6b519", size = 1868854, upload-time = "2025-05-12T11:54:21.564Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/b1/11a5453f8fcebe90a456471fad48bd154c6a62fcb96e3475a5e408d05fc8/gevent-25.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bb673eb291c19370f69295f7a881a536451408481e2e3deec3f41dedb7c281ec", size = 1833946, upload-time = "2025-05-12T12:00:05.514Z" },
+    { url = "https://files.pythonhosted.org/packages/70/1c/37d4a62303f86e6af67660a8df38c1171b7290df61b358e618c6fea79567/gevent-25.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1325ed44225c8309c0dd188bdbbbee79e1df8c11ceccac226b861c7d52e4837", size = 2070583, upload-time = "2025-05-12T11:33:02.803Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/8f/3b14929ff28263aba1d268ea97bcf104be1a86ba6f6bb4633838e7a1905e/gevent-25.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fcd5bcad3102bde686d0adcc341fade6245186050ce14386d547ccab4bd54310", size = 1808341, upload-time = "2025-05-12T11:59:59.154Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/fc/674ec819fb8a96e482e4d21f8baa43d34602dba09dfce7bbdc8700899d1b/gevent-25.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1a93062609e8fa67ec97cd5fb9206886774b2a09b24887f40148c9c37e6fb71c", size = 2137974, upload-time = "2025-05-12T11:40:54.78Z" },
+    { url = "https://files.pythonhosted.org/packages/05/9a/048b7f5e28c54e4595ad4a8ad3c338fa89560e558db2bbe8273f44f030de/gevent-25.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:2534c23dc32bed62b659ed4fd9e198906179e68b26c9276a897e04163bdde806", size = 1638344, upload-time = "2025-05-12T12:08:31.776Z" },
+    { url = "https://files.pythonhosted.org/packages/10/25/2162b38d7b48e08865db6772d632bd1648136ce2bb50e340565e45607cad/gevent-25.5.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a022a9de9275ce0b390b7315595454258c525dc8287a03f1a6cacc5878ab7cbc", size = 2928044, upload-time = "2025-05-12T11:11:36.33Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/e0/dbd597a964ed00176da122ea759bf2a6c1504f1e9f08e185379f92dc355f/gevent-25.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fae8533f9d0ef3348a1f503edcfb531ef7a0236b57da1e24339aceb0ce52922", size = 1788751, upload-time = "2025-05-12T11:52:32.643Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/74/960cc4cf4c9c90eafbe0efc238cdf588862e8e278d0b8c0d15a0da4ed480/gevent-25.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c7b32d9c3b5294b39ea9060e20c582e49e1ec81edbfeae6cf05f8ad0829cb13d", size = 1869766, upload-time = "2025-05-12T11:54:23.903Z" },
+    { url = "https://files.pythonhosted.org/packages/56/78/fa84b1c7db79b156929685db09a7c18c3127361dca18a09e998e98118506/gevent-25.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b95815fe44f318ebbfd733b6428b4cb18cc5e68f1c40e8501dd69cc1f42a83d", size = 1835358, upload-time = "2025-05-12T12:00:06.794Z" },
+    { url = "https://files.pythonhosted.org/packages/00/5c/bfefe3822bbca5b83bfad256c82251b3f5be13d52d14e17a786847b9b625/gevent-25.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d316529b70d325b183b2f3f5cde958911ff7be12eb2b532b5c301f915dbbf1e", size = 2073071, upload-time = "2025-05-12T11:33:04.2Z" },
+    { url = "https://files.pythonhosted.org/packages/20/e4/08a77a3839a37db96393dea952e992d5846a881b887986dde62ead6b48a1/gevent-25.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f6ba33c13db91ffdbb489a4f3d177a261ea1843923e1d68a5636c53fe98fa5ce", size = 1809805, upload-time = "2025-05-12T12:00:00.537Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/ac/28848348f790c1283df74b0fc0a554271d0606676470f848eccf84eae42a/gevent-25.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:37ee34b77c7553777c0b8379915f75934c3f9c8cd32f7cd098ea43c9323c2276", size = 2138305, upload-time = "2025-05-12T11:40:56.566Z" },
+    { url = "https://files.pythonhosted.org/packages/52/9e/0e9e40facd2d714bfb00f71fc6dacaacc82c24c1c2e097bf6461e00dec9f/gevent-25.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:9fa6aa0da224ed807d3b76cdb4ee8b54d4d4d5e018aed2478098e685baae7896", size = 1637444, upload-time = "2025-05-12T12:17:45.995Z" },
+    { url = "https://files.pythonhosted.org/packages/60/16/b71171e97ec7b4ded8669542f4369d88d5a289e2704efbbde51e858e062a/gevent-25.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0bacf89a65489d26c7087669af89938d5bfd9f7afb12a07b57855b9fad6ccbd0", size = 2937113, upload-time = "2025-05-12T11:12:03.191Z" },
+]
+
+[[package]]
+name = "geventhttpclient"
+version = "2.3.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "brotli" },
+    { name = "certifi" },
+    { name = "gevent" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/89/19/1ca8de73dcc0596d3df01be299e940d7fc3bccbeb6f62bb8dd2d427a3a50/geventhttpclient-2.3.4.tar.gz", hash = "sha256:1749f75810435a001fc6d4d7526c92cf02b39b30ab6217a886102f941c874222", size = 83545, upload-time = "2025-06-11T13:18:14.144Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4f/72/dcbc6dbf838549b7b0c2c18c1365d2580eb7456939e4b608c3ab213fce78/geventhttpclient-2.3.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9ac30c38d86d888b42bb2ab2738ab9881199609e9fa9a153eb0c66fc9188c6cb", size = 71984, upload-time = "2025-06-11T13:17:09.126Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/f9/74aa8c556364ad39b238919c954a0da01a6154ad5e85a1d1ab5f9f5ac186/geventhttpclient-2.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b802000a4fad80fa57e895009671d6e8af56777e3adf0d8aee0807e96188fd9", size = 52631, upload-time = "2025-06-11T13:17:10.061Z" },
+    { url = "https://files.pythonhosted.org/packages/11/1a/bc4b70cba8b46be8b2c6ca5b8067c4f086f8c90915eb68086ab40ff6243d/geventhttpclient-2.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:461e4d9f4caee481788ec95ac64e0a4a087c1964ddbfae9b6f2dc51715ba706c", size = 51991, upload-time = "2025-06-11T13:17:11.049Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/f5/8d0f1e998f6d933c251b51ef92d11f7eb5211e3cd579018973a2b455f7c5/geventhttpclient-2.3.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41f2dcc0805551ea9d49f9392c3b9296505a89b9387417b148655d0d8251b36e", size = 119012, upload-time = "2025-06-11T13:17:11.956Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/0e/59e4ab506b3c19fc72e88ca344d150a9028a00c400b1099637100bec26fc/geventhttpclient-2.3.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62f3a29bf242ecca6360d497304900683fd8f42cbf1de8d0546c871819251dad", size = 124565, upload-time = "2025-06-11T13:17:12.896Z" },
+    { url = "https://files.pythonhosted.org/packages/39/5d/dcbd34dfcda0c016b4970bd583cb260cc5ebfc35b33d0ec9ccdb2293587a/geventhttpclient-2.3.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8714a3f2c093aeda3ffdb14c03571d349cb3ed1b8b461d9f321890659f4a5dbf", size = 115573, upload-time = "2025-06-11T13:17:13.937Z" },
+    { url = "https://files.pythonhosted.org/packages/03/51/89af99e4805e9ce7f95562dfbd23c0b0391830831e43d58f940ec74489ac/geventhttpclient-2.3.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b11f38b74bab75282db66226197024a731250dcbe25542fd4e85ac5313547332", size = 114260, upload-time = "2025-06-11T13:17:14.913Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/ec/3a3000bda432953abcc6f51d008166fa7abc1eeddd1f0246933d83854f73/geventhttpclient-2.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fccc2023a89dfbce2e1b1409b967011e45d41808df81b7fa0259397db79ba647", size = 111592, upload-time = "2025-06-11T13:17:15.879Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/a3/88fd71fe6bbe1315a2d161cbe2cc7810c357d99bced113bea1668ede8bcf/geventhttpclient-2.3.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9d54b8e9a44890159ae36ba4ae44efd8bb79ff519055137a340d357538a68aa3", size = 113216, upload-time = "2025-06-11T13:17:16.883Z" },
+    { url = "https://files.pythonhosted.org/packages/52/eb/20435585a6911b26e65f901a827ef13551c053133926f8c28a7cca0fb08e/geventhttpclient-2.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:407cb68a3c3a2c4f5d503930298f2b26ae68137d520e8846d8e230a9981d9334", size = 118450, upload-time = "2025-06-11T13:17:17.968Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/79/82782283d613570373990b676a0966c1062a38ca8f41a0f20843c5808e01/geventhttpclient-2.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:54fbbcca2dcf06f12a337dd8f98417a09a49aa9d9706aa530fc93acb59b7d83c", size = 112226, upload-time = "2025-06-11T13:17:18.942Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/c4/417d12fc2a31ad93172b03309c7f8c3a8bbd0cf25b95eb7835de26b24453/geventhttpclient-2.3.4-cp312-cp312-win32.whl", hash = "sha256:83143b41bde2eb010c7056f142cb764cfbf77f16bf78bda2323a160767455cf5", size = 48365, upload-time = "2025-06-11T13:17:20.096Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/f4/7e5ee2f460bbbd09cb5d90ff63a1cf80d60f1c60c29dac20326324242377/geventhttpclient-2.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:46eda9a9137b0ca7886369b40995d2a43a5dff033d0a839a54241015d1845d41", size = 48961, upload-time = "2025-06-11T13:17:21.111Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/ad/132fddde6e2dca46d6a86316962437acd2bfaeb264db4e0fae83c529eb04/geventhttpclient-2.3.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:be64c5583884c407fc748dedbcb083475d5b138afb23c6bc0836cbad228402cc", size = 71967, upload-time = "2025-06-11T13:17:22.121Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/34/5e77d9a31d93409a8519cf573843288565272ae5a016be9c9293f56c50a1/geventhttpclient-2.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:15b2567137734183efda18e4d6245b18772e648b6a25adea0eba8b3a8b0d17e8", size = 52632, upload-time = "2025-06-11T13:17:23.016Z" },
+    { url = "https://files.pythonhosted.org/packages/47/d2/cf0dbc333304700e68cee9347f654b56e8b0f93a341b8b0d027ee96800d6/geventhttpclient-2.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a4bca1151b8cd207eef6d5cb3c720c562b2aa7293cf113a68874e235cfa19c31", size = 51980, upload-time = "2025-06-11T13:17:23.933Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/5b/c0c30ccd9d06c603add3f2d6abd68bd98430ee9730dc5478815759cf07f7/geventhttpclient-2.3.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b50d9daded5d36193d67e2fc30e59752262fcbbdc86e8222c7df6b93af0346a", size = 118987, upload-time = "2025-06-11T13:17:24.97Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/56/095a46af86476372064128162eccbd2ba4a7721503759890d32ea701d5fd/geventhttpclient-2.3.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe705e7656bc6982a463a4ed7f9b1db8c78c08323f1d45d0d1d77063efa0ce96", size = 124519, upload-time = "2025-06-11T13:17:25.933Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/12/7c9ba94b58f7954a83d33183152ce6bf5bda10c08ebe47d79a314cd33e29/geventhttpclient-2.3.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:69668589359db4cbb9efa327dda5735d1e74145e6f0a9ffa50236d15cf904053", size = 115574, upload-time = "2025-06-11T13:17:27.331Z" },
+    { url = "https://files.pythonhosted.org/packages/73/77/c4e7c5bce0199428fdb811d6adf6e347180d89eaa1b9b723f711f6bbc830/geventhttpclient-2.3.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9ba526e07ccaf4f1c2cd3395dda221139f01468b6eee1190d4a616f187a0378", size = 114222, upload-time = "2025-06-11T13:17:28.289Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/79/58802d300950dbd7d4e31eb24afd7c270fc7900ff3923fd266cc915bb086/geventhttpclient-2.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:525bd192705b5cb41a7cc3fe41fca194bfd6b5b59997ab9fe68fe0a82dab6140", size = 111682, upload-time = "2025-06-11T13:17:29.291Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/9c/ae04e4033459b8142788dad80d8d0b42d460bc6db9150e0815c2d0a02cb4/geventhttpclient-2.3.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:42b6f6afb0d3aab6a013c9cdb97e19bf4fe08695975670d0a018113d24cb344c", size = 113252, upload-time = "2025-06-11T13:17:30.357Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/67/5ae5d5878b06397a7b54334d1d31bb78cefc950ae890c2b8f5c917eb271e/geventhttpclient-2.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:227579b703085c4e5c6d5217ad6565b19ac8d1164404133e5874efaae1905114", size = 118426, upload-time = "2025-06-11T13:17:31.363Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/36/9065bb51f261950c42eddf8718e01a9ff344d8082e31317a8b6677be9bd6/geventhttpclient-2.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d1d0db89c1c8f3282eac9a22fda2b4082e1ed62a2107f70e3f1de1872c7919f", size = 112245, upload-time = "2025-06-11T13:17:32.331Z" },
+    { url = "https://files.pythonhosted.org/packages/21/7e/08a615bec095c288f997951e42e48b262d43c6081bef33cfbfad96ab9658/geventhttpclient-2.3.4-cp313-cp313-win32.whl", hash = "sha256:4e492b9ab880f98f8a9cc143b96ea72e860946eae8ad5fb2837cede2a8f45154", size = 48360, upload-time = "2025-06-11T13:17:33.349Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/19/ef3cb21e7e95b14cfcd21e3ba7fe3d696e171682dfa43ab8c0a727cac601/geventhttpclient-2.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:72575c5b502bf26ececccb905e4e028bb922f542946be701923e726acf305eb6", size = 48956, upload-time = "2025-06-11T13:17:34.956Z" },
+]
+
 [[package]]
 name = "gitdb"
 version = "4.0.12"
@@ -1342,6 +1512,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" },
 ]
 
+[[package]]
+name = "itsdangerous"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" },
+]
+
 [[package]]
 name = "jedi"
 version = "0.19.2"
@@ -1580,6 +1759,9 @@ ui = [
 ]
 
 [package.dev-dependencies]
+benchmark = [
+    { name = "locust" },
+]
 codegen = [
     { name = "jinja2" },
     { name = "pydantic" },
@@ -1695,6 +1877,7 @@ requires-dist = [
 provides-extras = ["ui"]
 
 [package.metadata.requires-dev]
+benchmark = [{ name = "locust", specifier = ">=2.37.14" }]
 codegen = [
     { name = "jinja2", specifier = ">=3.1.6" },
     { name = "pydantic" },
@@ -1800,6 +1983,47 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/81/fc/5eccc86b83c5ced3a3bca071d250a86ccafa4ff17546cf781deb7758ab74/llama_stack_client-0.2.17-py3-none-any.whl", hash = "sha256:336c32f8688700ff64717b8109f405dc87a990fbe310c2027ac9ed6d39d67d16", size = 350329, upload-time = "2025-08-05T01:42:54.381Z" },
 ]
 
+[[package]]
+name = "locust"
+version = "2.38.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "configargparse" },
+    { name = "flask" },
+    { name = "flask-cors" },
+    { name = "flask-login" },
+    { name = "gevent" },
+    { name = "geventhttpclient" },
+    { name = "locust-cloud" },
+    { name = "msgpack" },
+    { name = "psutil" },
+    { name = "pywin32", marker = "sys_platform == 'win32'" },
+    { name = "pyzmq" },
+    { name = "requests" },
+    { name = "setuptools" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fb/93/ecd79dde28e24bdc99488d4e2c0ad4117252257d5cbdd61e3b14d1f03786/locust-2.38.0.tar.gz", hash = "sha256:5bd6c29d8423733cb5d9a265548c9fef7b731f2254aa91885d6c98d0d39f90f0", size = 1406518, upload-time = "2025-08-07T10:18:52.584Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ae/be/57ca67b95c45e69c173e86fe5c934d789effc2ec203d3e3ec2a0b32aa707/locust-2.38.0-py3-none-any.whl", hash = "sha256:b92c937e8659e9ffd6d6d1cab2f63f70aa98c87975911938d1f473534f46fd78", size = 1424083, upload-time = "2025-08-07T10:18:50.499Z" },
+]
+
+[[package]]
+name = "locust-cloud"
+version = "1.26.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "configargparse" },
+    { name = "gevent" },
+    { name = "platformdirs" },
+    { name = "python-engineio" },
+    { name = "python-socketio", extra = ["client"] },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/84/ad/10b299b134068a4250a9156e6832a717406abe1dfea2482a07ae7bdca8f3/locust_cloud-1.26.3.tar.gz", hash = "sha256:587acfd4d2dee715fb5f0c3c2d922770babf0b7cff7b2927afbb693a9cd193cc", size = 456042, upload-time = "2025-07-15T19:51:53.791Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/50/6a/276fc50a9d170e7cbb6715735480cb037abb526639bca85491576e6eee4a/locust_cloud-1.26.3-py3-none-any.whl", hash = "sha256:8cb4b8bb9adcd5b99327bc8ed1d98cf67a29d9d29512651e6e94869de6f1faa8", size = 410023, upload-time = "2025-07-15T19:51:52.056Z" },
+]
+
 [[package]]
 name = "lxml"
 version = "6.0.0"
@@ -2017,6 +2241,34 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" },
 ]
 
+[[package]]
+name = "msgpack"
+version = "1.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" },
+    { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" },
+    { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/38/561f01cf3577430b59b340b51329803d3a5bf6a45864a55f4ef308ac11e3/msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0", size = 81677, upload-time = "2025-06-13T06:52:16.64Z" },
+    { url = "https://files.pythonhosted.org/packages/09/48/54a89579ea36b6ae0ee001cba8c61f776451fad3c9306cd80f5b5c55be87/msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9", size = 78603, upload-time = "2025-06-13T06:52:17.843Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/60/daba2699b308e95ae792cdc2ef092a38eb5ee422f9d2fbd4101526d8a210/msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8", size = 420504, upload-time = "2025-06-13T06:52:18.982Z" },
+    { url = "https://files.pythonhosted.org/packages/20/22/2ebae7ae43cd8f2debc35c631172ddf14e2a87ffcc04cf43ff9df9fff0d3/msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a", size = 423749, upload-time = "2025-06-13T06:52:20.211Z" },
+    { url = "https://files.pythonhosted.org/packages/40/1b/54c08dd5452427e1179a40b4b607e37e2664bca1c790c60c442c8e972e47/msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac", size = 404458, upload-time = "2025-06-13T06:52:21.429Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/60/6bb17e9ffb080616a51f09928fdd5cac1353c9becc6c4a8abd4e57269a16/msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b", size = 405976, upload-time = "2025-06-13T06:52:22.995Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/97/88983e266572e8707c1f4b99c8fd04f9eb97b43f2db40e3172d87d8642db/msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7", size = 408607, upload-time = "2025-06-13T06:52:24.152Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/66/36c78af2efaffcc15a5a61ae0df53a1d025f2680122e2a9eb8442fed3ae4/msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5", size = 424172, upload-time = "2025-06-13T06:52:25.704Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/87/a75eb622b555708fe0427fab96056d39d4c9892b0c784b3a721088c7ee37/msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323", size = 65347, upload-time = "2025-06-13T06:52:26.846Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" },
+]
+
 [[package]]
 name = "multidict"
 version = "6.6.3"
@@ -3306,6 +3558,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5f/ed/539768cf28c661b5b068d66d96a2f155c4971a5d55684a514c1a0e0dec2f/python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc", size = 20556, upload-time = "2025-06-24T04:21:06.073Z" },
 ]
 
+[[package]]
+name = "python-engineio"
+version = "4.12.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "simple-websocket" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ba/0b/67295279b66835f9fa7a491650efcd78b20321c127036eef62c11a31e028/python_engineio-4.12.2.tar.gz", hash = "sha256:e7e712ffe1be1f6a05ee5f951e72d434854a32fcfc7f6e4d9d3cae24ec70defa", size = 91677, upload-time = "2025-06-04T19:22:18.789Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0c/fa/df59acedf7bbb937f69174d00f921a7b93aa5a5f5c17d05296c814fff6fc/python_engineio-4.12.2-py3-none-any.whl", hash = "sha256:8218ab66950e179dfec4b4bbb30aecf3f5d86f5e58e6fc1aa7fde2c698b2804f", size = 59536, upload-time = "2025-06-04T19:22:16.916Z" },
+]
+
 [[package]]
 name = "python-jose"
 version = "3.5.0"
@@ -3334,6 +3598,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" },
 ]
 
+[[package]]
+name = "python-socketio"
+version = "5.13.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "bidict" },
+    { name = "python-engineio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/21/1a/396d50ccf06ee539fa758ce5623b59a9cb27637fc4b2dc07ed08bf495e77/python_socketio-5.13.0.tar.gz", hash = "sha256:ac4e19a0302ae812e23b712ec8b6427ca0521f7c582d6abb096e36e24a263029", size = 121125, upload-time = "2025-04-12T15:46:59.933Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3c/32/b4fb8585d1be0f68bde7e110dffbcf354915f77ad8c778563f0ad9655c02/python_socketio-5.13.0-py3-none-any.whl", hash = "sha256:51f68d6499f2df8524668c24bcec13ba1414117cfb3a90115c559b601ab10caf", size = 77800, upload-time = "2025-04-12T15:46:58.412Z" },
+]
+
+[package.optional-dependencies]
+client = [
+    { name = "requests" },
+    { name = "websocket-client" },
+]
+
 [[package]]
 name = "pytz"
 version = "2025.2"
@@ -3726,6 +4009,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
 ]
 
+[[package]]
+name = "simple-websocket"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "wsproto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" },
+]
+
 [[package]]
 name = "six"
 version = "1.17.0"
@@ -4796,6 +5091,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" },
 ]
 
+[[package]]
+name = "werkzeug"
+version = "3.1.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markupsafe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
+]
+
+[[package]]
+name = "wsproto"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/4a/44d3c295350d776427904d73c189e10aeae66d7f555bb2feee16d1e4ba5a/wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", size = 53425, upload-time = "2022-08-23T19:58:21.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" },
+]
+
 [[package]]
 name = "xxhash"
 version = "3.5.0"
@@ -4907,3 +5226,38 @@ sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50e
 wheels = [
     { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
 ]
+
+[[package]]
+name = "zope-event"
+version = "5.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "setuptools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5a/9f/c443569a68d3844c044d9fa9711e08adb33649b527b4d432433f4c2a6a02/zope_event-5.1.1.tar.gz", hash = "sha256:c1ac931abf57efba71a2a313c5f4d57768a19b15c37e3f02f50eb1536be12d4e", size = 18811, upload-time = "2025-07-22T07:04:00.924Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e9/04/fd55695f6448abd22295fc68b2d3a135389558f0f49a24b0dffe019d0ecb/zope_event-5.1.1-py3-none-any.whl", hash = "sha256:8d5ea7b992c42ce73a6fa9c2ba99a004c52cd9f05d87f3220768ef0329b92df7", size = 7014, upload-time = "2025-07-22T07:03:59.9Z" },
+]
+
+[[package]]
+name = "zope-interface"
+version = "7.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "setuptools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" },
+    { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" },
+    { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/3b/e309d731712c1a1866d61b5356a069dd44e5b01e394b6cb49848fa2efbff/zope.interface-7.2-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:3e0350b51e88658d5ad126c6a57502b19d5f559f6cb0a628e3dc90442b53dd98", size = 208961, upload-time = "2024-11-28T08:48:29.865Z" },
+    { url = "https://files.pythonhosted.org/packages/49/65/78e7cebca6be07c8fc4032bfbb123e500d60efdf7b86727bb8a071992108/zope.interface-7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15398c000c094b8855d7d74f4fdc9e73aa02d4d0d5c775acdef98cdb1119768d", size = 209356, upload-time = "2024-11-28T08:48:33.297Z" },
+    { url = "https://files.pythonhosted.org/packages/11/b1/627384b745310d082d29e3695db5f5a9188186676912c14b61a78bbc6afe/zope.interface-7.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:802176a9f99bd8cc276dcd3b8512808716492f6f557c11196d42e26c01a69a4c", size = 264196, upload-time = "2024-11-28T09:18:17.584Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/f6/54548df6dc73e30ac6c8a7ff1da73ac9007ba38f866397091d5a82237bd3/zope.interface-7.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb23f58a446a7f09db85eda09521a498e109f137b85fb278edb2e34841055398", size = 259237, upload-time = "2024-11-28T08:48:31.71Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/66/ac05b741c2129fdf668b85631d2268421c5cd1a9ff99be1674371139d665/zope.interface-7.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a71a5b541078d0ebe373a81a3b7e71432c61d12e660f1d67896ca62d9628045b", size = 264696, upload-time = "2024-11-28T08:48:41.161Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/2f/1bccc6f4cc882662162a1158cda1a7f616add2ffe322b28c99cb031b4ffc/zope.interface-7.2-cp313-cp313-win_amd64.whl", hash = "sha256:4893395d5dd2ba655c38ceb13014fd65667740f09fa5bb01caa1e6284e48c0cd", size = 212472, upload-time = "2024-11-28T08:49:56.587Z" },
+]

From 5b312a80b99beebb87e5414fb28940806aa5d347 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Wed, 13 Aug 2025 11:23:27 -0700
Subject: [PATCH 21/45] feat(responses): improve streaming for function calls
 (#3124)

Emit streaming events for function calls

## Test Plan

Improved the test case
---
 .../agents/meta_reference/openai_responses.py | 152 +++++++++++++++---
 .../non_ci/responses/test_responses.py        | 113 ++++++++++++-
 .../meta_reference/test_openai_responses.py   |  18 ++-
 3 files changed, 250 insertions(+), 33 deletions(-)

diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
index 347954908..104f15010 100644
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
@@ -33,6 +33,10 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseObjectStream,
     OpenAIResponseObjectStreamResponseCompleted,
     OpenAIResponseObjectStreamResponseCreated,
+    OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta,
+    OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone,
+    OpenAIResponseObjectStreamResponseOutputItemAdded,
+    OpenAIResponseObjectStreamResponseOutputItemDone,
     OpenAIResponseObjectStreamResponseOutputTextDelta,
     OpenAIResponseOutput,
     OpenAIResponseOutputMessageContent,
@@ -73,7 +77,9 @@ from llama_stack.apis.tools import ToolGroups, ToolInvocationResult, ToolRuntime
 from llama_stack.apis.vector_io import VectorIO
 from llama_stack.log import get_logger
 from llama_stack.models.llama.datatypes import ToolDefinition, ToolParamDefinition
-from llama_stack.providers.utils.inference.openai_compat import convert_tooldef_to_openai_tool
+from llama_stack.providers.utils.inference.openai_compat import (
+    convert_tooldef_to_openai_tool,
+)
 from llama_stack.providers.utils.responses.responses_store import ResponsesStore
 
 logger = get_logger(name=__name__, category="openai_responses")
@@ -82,7 +88,7 @@ OPENAI_RESPONSES_PREFIX = "openai_responses:"
 
 
 async def _convert_response_content_to_chat_content(
-    content: str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent],
+    content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
 ) -> str | list[OpenAIChatCompletionContentPartParam]:
     """
     Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts.
@@ -150,7 +156,9 @@ async def _convert_response_input_to_chat_messages(
     return messages
 
 
-async def _convert_chat_choice_to_response_message(choice: OpenAIChoice) -> OpenAIResponseMessage:
+async def _convert_chat_choice_to_response_message(
+    choice: OpenAIChoice,
+) -> OpenAIResponseMessage:
     """
     Convert an OpenAI Chat Completion choice into an OpenAI Response output message.
     """
@@ -172,7 +180,9 @@ async def _convert_chat_choice_to_response_message(choice: OpenAIChoice) -> Open
     )
 
 
-async def _convert_response_text_to_chat_response_format(text: OpenAIResponseText) -> OpenAIResponseFormatParam:
+async def _convert_response_text_to_chat_response_format(
+    text: OpenAIResponseText,
+) -> OpenAIResponseFormatParam:
     """
     Convert an OpenAI Response text parameter into an OpenAI Chat Completion response format.
     """
@@ -228,7 +238,9 @@ class OpenAIResponsesImpl:
         self.vector_io_api = vector_io_api
 
     async def _prepend_previous_response(
-        self, input: str | list[OpenAIResponseInput], previous_response_id: str | None = None
+        self,
+        input: str | list[OpenAIResponseInput],
+        previous_response_id: str | None = None,
     ):
         if previous_response_id:
             previous_response_with_input = await self.responses_store.get_response_object(previous_response_id)
@@ -446,6 +458,8 @@ class OpenAIResponsesImpl:
 
             # Create a placeholder message item for delta events
             message_item_id = f"msg_{uuid.uuid4()}"
+            # Track tool call items for streaming events
+            tool_call_item_ids: dict[int, str] = {}
 
             async for chunk in completion_result:
                 chat_response_id = chunk.id
@@ -472,18 +486,62 @@ class OpenAIResponsesImpl:
                     if chunk_choice.delta.tool_calls:
                         for tool_call in chunk_choice.delta.tool_calls:
                             response_tool_call = chat_response_tool_calls.get(tool_call.index, None)
-                            if response_tool_call:
-                                # Don't attempt to concatenate arguments if we don't have any new argumentsAdd commentMore actions
-                                if tool_call.function.arguments:
-                                    # Guard against an initial None argument before we concatenate
-                                    response_tool_call.function.arguments = (
-                                        response_tool_call.function.arguments or ""
-                                    ) + tool_call.function.arguments
-                            else:
+                            # Create new tool call entry if this is the first chunk for this index
+                            is_new_tool_call = response_tool_call is None
+                            if is_new_tool_call:
                                 tool_call_dict: dict[str, Any] = tool_call.model_dump()
                                 tool_call_dict.pop("type", None)
                                 response_tool_call = OpenAIChatCompletionToolCall(**tool_call_dict)
-                            chat_response_tool_calls[tool_call.index] = response_tool_call
+                                chat_response_tool_calls[tool_call.index] = response_tool_call
+
+                                # Create item ID for this tool call for streaming events
+                                tool_call_item_id = f"fc_{uuid.uuid4()}"
+                                tool_call_item_ids[tool_call.index] = tool_call_item_id
+
+                                # Emit output_item.added event for the new function call
+                                sequence_number += 1
+                                function_call_item = OpenAIResponseOutputMessageFunctionToolCall(
+                                    arguments="",  # Will be filled incrementally via delta events
+                                    call_id=tool_call.id or "",
+                                    name=tool_call.function.name if tool_call.function else "",
+                                    id=tool_call_item_id,
+                                    status="in_progress",
+                                )
+                                yield OpenAIResponseObjectStreamResponseOutputItemAdded(
+                                    response_id=response_id,
+                                    item=function_call_item,
+                                    output_index=len(output_messages),
+                                    sequence_number=sequence_number,
+                                )
+
+                            # Stream function call arguments as they arrive
+                            if tool_call.function and tool_call.function.arguments:
+                                tool_call_item_id = tool_call_item_ids[tool_call.index]
+                                sequence_number += 1
+                                yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta(
+                                    delta=tool_call.function.arguments,
+                                    item_id=tool_call_item_id,
+                                    output_index=len(output_messages),
+                                    sequence_number=sequence_number,
+                                )
+
+                                # Accumulate arguments for final response (only for subsequent chunks)
+                                if not is_new_tool_call:
+                                    response_tool_call.function.arguments = (
+                                        response_tool_call.function.arguments or ""
+                                    ) + tool_call.function.arguments
+
+            # Emit function_call_arguments.done events for completed tool calls
+            for tool_call_index in sorted(chat_response_tool_calls.keys()):
+                tool_call_item_id = tool_call_item_ids[tool_call_index]
+                final_arguments = chat_response_tool_calls[tool_call_index].function.arguments or ""
+                sequence_number += 1
+                yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone(
+                    arguments=final_arguments,
+                    item_id=tool_call_item_id,
+                    output_index=len(output_messages),
+                    sequence_number=sequence_number,
+                )
 
             # Convert collected chunks to complete response
             if chat_response_tool_calls:
@@ -532,18 +590,56 @@ class OpenAIResponsesImpl:
                 tool_call_log, tool_response_message = await self._execute_tool_call(tool_call, ctx)
                 if tool_call_log:
                     output_messages.append(tool_call_log)
+
+                    # Emit output_item.done event for completed non-function tool call
+                    # Find the item_id for this tool call
+                    matching_item_id = None
+                    for index, item_id in tool_call_item_ids.items():
+                        response_tool_call = chat_response_tool_calls.get(index)
+                        if response_tool_call and response_tool_call.id == tool_call.id:
+                            matching_item_id = item_id
+                            break
+
+                    if matching_item_id:
+                        sequence_number += 1
+                        yield OpenAIResponseObjectStreamResponseOutputItemDone(
+                            response_id=response_id,
+                            item=tool_call_log,
+                            output_index=len(output_messages) - 1,
+                            sequence_number=sequence_number,
+                        )
+
                 if tool_response_message:
                     next_turn_messages.append(tool_response_message)
 
             for tool_call in function_tool_calls:
-                output_messages.append(
-                    OpenAIResponseOutputMessageFunctionToolCall(
-                        arguments=tool_call.function.arguments or "",
-                        call_id=tool_call.id,
-                        name=tool_call.function.name or "",
-                        id=f"fc_{uuid.uuid4()}",
-                        status="completed",
-                    )
+                # Find the item_id for this tool call from our tracking dictionary
+                matching_item_id = None
+                for index, item_id in tool_call_item_ids.items():
+                    response_tool_call = chat_response_tool_calls.get(index)
+                    if response_tool_call and response_tool_call.id == tool_call.id:
+                        matching_item_id = item_id
+                        break
+
+                # Use existing item_id or create new one if not found
+                final_item_id = matching_item_id or f"fc_{uuid.uuid4()}"
+
+                function_call_item = OpenAIResponseOutputMessageFunctionToolCall(
+                    arguments=tool_call.function.arguments or "",
+                    call_id=tool_call.id,
+                    name=tool_call.function.name or "",
+                    id=final_item_id,
+                    status="completed",
+                )
+                output_messages.append(function_call_item)
+
+                # Emit output_item.done event for completed function call
+                sequence_number += 1
+                yield OpenAIResponseObjectStreamResponseOutputItemDone(
+                    response_id=response_id,
+                    item=function_call_item,
+                    output_index=len(output_messages) - 1,
+                    sequence_number=sequence_number,
                 )
 
             if not function_tool_calls and not non_function_tool_calls:
@@ -779,7 +875,8 @@ class OpenAIResponsesImpl:
                 )
             elif function.name == "knowledge_search":
                 response_file_search_tool = next(
-                    (t for t in ctx.response_tools if isinstance(t, OpenAIResponseInputToolFileSearch)), None
+                    (t for t in ctx.response_tools if isinstance(t, OpenAIResponseInputToolFileSearch)),
+                    None,
                 )
                 if response_file_search_tool:
                     # Use vector_stores.search API instead of knowledge_search tool
@@ -798,7 +895,9 @@ class OpenAIResponsesImpl:
             error_exc = e
 
         if function.name in ctx.mcp_tool_to_server:
-            from llama_stack.apis.agents.openai_responses import OpenAIResponseOutputMessageMCPCall
+            from llama_stack.apis.agents.openai_responses import (
+                OpenAIResponseOutputMessageMCPCall,
+            )
 
             message = OpenAIResponseOutputMessageMCPCall(
                 id=tool_call_id,
@@ -850,7 +949,10 @@ class OpenAIResponsesImpl:
             if isinstance(result.content, str):
                 content = result.content
             elif isinstance(result.content, list):
-                from llama_stack.apis.common.content_types import ImageContentItem, TextContentItem
+                from llama_stack.apis.common.content_types import (
+                    ImageContentItem,
+                    TextContentItem,
+                )
 
                 content = []
                 for item in result.content:
diff --git a/tests/integration/non_ci/responses/test_responses.py b/tests/integration/non_ci/responses/test_responses.py
index 39d00f328..6092346b0 100644
--- a/tests/integration/non_ci/responses/test_responses.py
+++ b/tests/integration/non_ci/responses/test_responses.py
@@ -384,12 +384,18 @@ def test_response_non_streaming_mcp_tool(request, compat_client, text_model_id,
         assert list_tools.type == "mcp_list_tools"
         assert list_tools.server_label == "localmcp"
         assert len(list_tools.tools) == 2
-        assert {t.name for t in list_tools.tools} == {"get_boiling_point", "greet_everyone"}
+        assert {t.name for t in list_tools.tools} == {
+            "get_boiling_point",
+            "greet_everyone",
+        }
 
         call = response.output[1]
         assert call.type == "mcp_call"
         assert call.name == "get_boiling_point"
-        assert json.loads(call.arguments) == {"liquid_name": "myawesomeliquid", "celsius": True}
+        assert json.loads(call.arguments) == {
+            "liquid_name": "myawesomeliquid",
+            "celsius": True,
+        }
         assert call.error is None
         assert "-100" in call.output
 
@@ -581,6 +587,105 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
             f"Last chunk should be response.completed, got {chunks[-1].type}"
         )
 
+        # Verify tool call streaming events are present
+        chunk_types = [chunk.type for chunk in chunks]
+
+        # Should have function call arguments delta events for tool calls
+        delta_events = [chunk for chunk in chunks if chunk.type == "response.function_call_arguments.delta"]
+        done_events = [chunk for chunk in chunks if chunk.type == "response.function_call_arguments.done"]
+
+        # Should have output item events for tool calls
+        item_added_events = [chunk for chunk in chunks if chunk.type == "response.output_item.added"]
+        item_done_events = [chunk for chunk in chunks if chunk.type == "response.output_item.done"]
+
+        # Verify we have substantial streaming activity (not just batch events)
+        assert len(chunks) > 10, f"Expected rich streaming with many events, got only {len(chunks)} chunks"
+
+        # Since this test involves MCP tool calls, we should see streaming events
+        assert len(delta_events) > 0, f"Expected function_call_arguments.delta events, got chunk types: {chunk_types}"
+        assert len(done_events) > 0, f"Expected function_call_arguments.done events, got chunk types: {chunk_types}"
+
+        # Should have output item events for function calls
+        assert len(item_added_events) > 0, f"Expected response.output_item.added events, got chunk types: {chunk_types}"
+        assert len(item_done_events) > 0, f"Expected response.output_item.done events, got chunk types: {chunk_types}"
+
+        # Verify delta events have proper structure
+        for delta_event in delta_events:
+            assert hasattr(delta_event, "delta"), "Delta event should have 'delta' field"
+            assert hasattr(delta_event, "item_id"), "Delta event should have 'item_id' field"
+            assert hasattr(delta_event, "sequence_number"), "Delta event should have 'sequence_number' field"
+            assert delta_event.delta, "Delta should not be empty"
+
+        # Verify done events have proper structure
+        for done_event in done_events:
+            assert hasattr(done_event, "arguments"), "Done event should have 'arguments' field"
+            assert hasattr(done_event, "item_id"), "Done event should have 'item_id' field"
+            assert done_event.arguments, "Final arguments should not be empty"
+
+        # Verify output item added events have proper structure
+        for added_event in item_added_events:
+            assert hasattr(added_event, "item"), "Added event should have 'item' field"
+            assert hasattr(added_event, "output_index"), "Added event should have 'output_index' field"
+            assert hasattr(added_event, "sequence_number"), "Added event should have 'sequence_number' field"
+            assert hasattr(added_event, "response_id"), "Added event should have 'response_id' field"
+            assert added_event.item.type in ["function_call", "mcp_call"], "Added item should be a tool call"
+            assert added_event.item.status == "in_progress", "Added item should be in progress"
+            assert added_event.response_id, "Response ID should not be empty"
+            assert isinstance(added_event.output_index, int), "Output index should be integer"
+            assert added_event.output_index >= 0, "Output index should be non-negative"
+
+        # Verify output item done events have proper structure
+        for done_event in item_done_events:
+            assert hasattr(done_event, "item"), "Done event should have 'item' field"
+            assert hasattr(done_event, "output_index"), "Done event should have 'output_index' field"
+            assert hasattr(done_event, "sequence_number"), "Done event should have 'sequence_number' field"
+            assert hasattr(done_event, "response_id"), "Done event should have 'response_id' field"
+            assert done_event.item.type in ["function_call", "mcp_call"], "Done item should be a tool call"
+            # Note: MCP calls don't have a status field, only function calls do
+            if done_event.item.type == "function_call":
+                assert done_event.item.status == "completed", "Function call should be completed"
+            assert done_event.response_id, "Response ID should not be empty"
+            assert isinstance(done_event.output_index, int), "Output index should be integer"
+            assert done_event.output_index >= 0, "Output index should be non-negative"
+
+        # Group function call argument events by item_id (these should have proper tracking)
+        function_call_events_by_item_id = {}
+        for chunk in chunks:
+            if hasattr(chunk, "item_id") and chunk.type in [
+                "response.function_call_arguments.delta",
+                "response.function_call_arguments.done",
+            ]:
+                item_id = chunk.item_id
+                if item_id not in function_call_events_by_item_id:
+                    function_call_events_by_item_id[item_id] = []
+                function_call_events_by_item_id[item_id].append(chunk)
+
+        for item_id, related_events in function_call_events_by_item_id.items():
+            # Should have at least one delta and one done event for a complete function call
+            delta_events = [e for e in related_events if e.type == "response.function_call_arguments.delta"]
+            done_events = [e for e in related_events if e.type == "response.function_call_arguments.done"]
+
+            assert len(delta_events) > 0, f"Item {item_id} should have at least one delta event"
+            assert len(done_events) == 1, f"Item {item_id} should have exactly one done event"
+
+            # Verify all events have the same item_id
+            for event in related_events:
+                assert event.item_id == item_id, f"Event should have consistent item_id {item_id}, got {event.item_id}"
+
+        # Basic pairing check: each output_item.added should be followed by some activity
+        # (but we can't enforce strict 1:1 pairing due to the complexity of multi-turn scenarios)
+        assert len(item_added_events) > 0, "Should have at least one output_item.added event"
+
+        # Verify response_id consistency across all events
+        response_ids = set()
+        for chunk in chunks:
+            if hasattr(chunk, "response_id"):
+                response_ids.add(chunk.response_id)
+            elif hasattr(chunk, "response") and hasattr(chunk.response, "id"):
+                response_ids.add(chunk.response.id)
+
+        assert len(response_ids) == 1, f"All events should reference the same response_id, found: {response_ids}"
+
         # Get the final response from the last chunk
         final_chunk = chunks[-1]
         if hasattr(final_chunk, "response"):
@@ -722,7 +827,9 @@ def vector_store_with_filtered_files(compat_client, text_model_id, tmp_path_fact
 
         # Attach file to vector store with attributes
         file_attach_response = compat_client.vector_stores.files.create(
-            vector_store_id=vector_store.id, file_id=file_response.id, attributes=file_data["attributes"]
+            vector_store_id=vector_store.id,
+            file_id=file_response.id,
+            attributes=file_data["attributes"],
         )
 
         # Wait for attachment
diff --git a/tests/unit/providers/agents/meta_reference/test_openai_responses.py b/tests/unit/providers/agents/meta_reference/test_openai_responses.py
index 2ab5b557e..855a525e9 100644
--- a/tests/unit/providers/agents/meta_reference/test_openai_responses.py
+++ b/tests/unit/providers/agents/meta_reference/test_openai_responses.py
@@ -272,7 +272,9 @@ async def test_create_openai_response_with_tool_call_type_none(openai_responses_
 
     # Check that we got the content from our mocked tool execution result
     chunks = [chunk async for chunk in result]
-    assert len(chunks) == 2  # Should have response.created and response.completed
+    # Should have: response.created, output_item.added, function_call_arguments.delta,
+    # function_call_arguments.done, output_item.done, response.completed
+    assert len(chunks) == 6
 
     # Verify inference API was called correctly (after iterating over result)
     first_call = mock_inference_api.openai_chat_completion.call_args_list[0]
@@ -284,11 +286,17 @@ async def test_create_openai_response_with_tool_call_type_none(openai_responses_
     assert chunks[0].type == "response.created"
     assert len(chunks[0].response.output) == 0
 
+    # Check streaming events
+    assert chunks[1].type == "response.output_item.added"
+    assert chunks[2].type == "response.function_call_arguments.delta"
+    assert chunks[3].type == "response.function_call_arguments.done"
+    assert chunks[4].type == "response.output_item.done"
+
     # Check response.completed event (should have the tool call)
-    assert chunks[1].type == "response.completed"
-    assert len(chunks[1].response.output) == 1
-    assert chunks[1].response.output[0].type == "function_call"
-    assert chunks[1].response.output[0].name == "get_weather"
+    assert chunks[5].type == "response.completed"
+    assert len(chunks[5].response.output) == 1
+    assert chunks[5].response.output[0].type == "function_call"
+    assert chunks[5].response.output[0].name == "get_weather"
 
 
 async def test_create_openai_response_with_multiple_messages(openai_responses_impl, mock_inference_api):

From 8638537d14f0dc4a0b3a1acdaa295894f205b83f Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Wed, 13 Aug 2025 16:31:25 -0700
Subject: [PATCH 22/45] feat(responses): stream progress of tool calls (#3135)

# What does this PR do?
Enhances tool execution streaming by adding support for real-time progress events during tool calls. This implementation adds streaming events for MCP and web search tools, including in-progress, searching, completed, and failed states.

The refactored `_execute_tool_call` method now returns an async iterator that yields streaming events throughout the tool execution lifecycle.

## Test Plan
Updated the integration test `test_response_streaming_multi_turn_tool_execution` to verify the presence and structure of new streaming events, including:
- Checking for MCP in-progress and completed events
- Verifying that progress events contain required fields (item_id, output_index, sequence_number)
- Ensuring completed events have the necessary sequence_number field
---
 .../agents/meta_reference/openai_responses.py | 137 +++++++++++++++---
 .../non_ci/responses/test_responses.py        |  22 +++
 2 files changed, 141 insertions(+), 18 deletions(-)

diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
index 104f15010..fbb5a608a 100644
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
@@ -35,9 +35,15 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseObjectStreamResponseCreated,
     OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta,
     OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone,
+    OpenAIResponseObjectStreamResponseMcpCallCompleted,
+    OpenAIResponseObjectStreamResponseMcpCallFailed,
+    OpenAIResponseObjectStreamResponseMcpCallInProgress,
     OpenAIResponseObjectStreamResponseOutputItemAdded,
     OpenAIResponseObjectStreamResponseOutputItemDone,
     OpenAIResponseObjectStreamResponseOutputTextDelta,
+    OpenAIResponseObjectStreamResponseWebSearchCallCompleted,
+    OpenAIResponseObjectStreamResponseWebSearchCallInProgress,
+    OpenAIResponseObjectStreamResponseWebSearchCallSearching,
     OpenAIResponseOutput,
     OpenAIResponseOutputMessageContent,
     OpenAIResponseOutputMessageContentOutputText,
@@ -87,6 +93,15 @@ logger = get_logger(name=__name__, category="openai_responses")
 OPENAI_RESPONSES_PREFIX = "openai_responses:"
 
 
+class ToolExecutionResult(BaseModel):
+    """Result of streaming tool execution."""
+
+    stream_event: OpenAIResponseObjectStream | None = None
+    sequence_number: int
+    final_output_message: OpenAIResponseOutput | None = None
+    final_input_message: OpenAIMessageParam | None = None
+
+
 async def _convert_response_content_to_chat_content(
     content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
 ) -> str | list[OpenAIChatCompletionContentPartParam]:
@@ -587,19 +602,38 @@ class OpenAIResponsesImpl:
 
             # execute non-function tool calls
             for tool_call in non_function_tool_calls:
-                tool_call_log, tool_response_message = await self._execute_tool_call(tool_call, ctx)
+                # Find the item_id for this tool call
+                matching_item_id = None
+                for index, item_id in tool_call_item_ids.items():
+                    response_tool_call = chat_response_tool_calls.get(index)
+                    if response_tool_call and response_tool_call.id == tool_call.id:
+                        matching_item_id = item_id
+                        break
+
+                # Use a fallback item_id if not found
+                if not matching_item_id:
+                    matching_item_id = f"tc_{uuid.uuid4()}"
+
+                # Execute tool call with streaming
+                tool_call_log = None
+                tool_response_message = None
+                async for result in self._execute_tool_call(
+                    tool_call, ctx, sequence_number, response_id, len(output_messages), matching_item_id
+                ):
+                    if result.stream_event:
+                        # Forward streaming events
+                        sequence_number = result.sequence_number
+                        yield result.stream_event
+
+                    if result.final_output_message is not None:
+                        tool_call_log = result.final_output_message
+                        tool_response_message = result.final_input_message
+                        sequence_number = result.sequence_number
+
                 if tool_call_log:
                     output_messages.append(tool_call_log)
 
                     # Emit output_item.done event for completed non-function tool call
-                    # Find the item_id for this tool call
-                    matching_item_id = None
-                    for index, item_id in tool_call_item_ids.items():
-                        response_tool_call = chat_response_tool_calls.get(index)
-                        if response_tool_call and response_tool_call.id == tool_call.id:
-                            matching_item_id = item_id
-                            break
-
                     if matching_item_id:
                         sequence_number += 1
                         yield OpenAIResponseObjectStreamResponseOutputItemDone(
@@ -848,7 +882,11 @@ class OpenAIResponsesImpl:
         self,
         tool_call: OpenAIChatCompletionToolCall,
         ctx: ChatCompletionContext,
-    ) -> tuple[OpenAIResponseOutput | None, OpenAIMessageParam | None]:
+        sequence_number: int,
+        response_id: str,
+        output_index: int,
+        item_id: str,
+    ) -> AsyncIterator[ToolExecutionResult]:
         from llama_stack.providers.utils.inference.prompt_adapter import (
             interleaved_content_as_str,
         )
@@ -858,8 +896,41 @@ class OpenAIResponsesImpl:
         tool_kwargs = json.loads(function.arguments) if function.arguments else {}
 
         if not function or not tool_call_id or not function.name:
-            return None, None
+            yield ToolExecutionResult(sequence_number=sequence_number)
+            return
 
+        # Emit in_progress event based on tool type (only for tools with specific streaming events)
+        progress_event = None
+        if ctx.mcp_tool_to_server and function.name in ctx.mcp_tool_to_server:
+            sequence_number += 1
+            progress_event = OpenAIResponseObjectStreamResponseMcpCallInProgress(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+        elif function.name == "web_search":
+            sequence_number += 1
+            progress_event = OpenAIResponseObjectStreamResponseWebSearchCallInProgress(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+        # Note: knowledge_search and other custom tools don't have specific streaming events in OpenAI spec
+
+        if progress_event:
+            yield ToolExecutionResult(stream_event=progress_event, sequence_number=sequence_number)
+
+        # For web search, emit searching event
+        if function.name == "web_search":
+            sequence_number += 1
+            searching_event = OpenAIResponseObjectStreamResponseWebSearchCallSearching(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+            yield ToolExecutionResult(stream_event=searching_event, sequence_number=sequence_number)
+
+        # Execute the actual tool call
         error_exc = None
         result = None
         try:
@@ -894,6 +965,33 @@ class OpenAIResponsesImpl:
         except Exception as e:
             error_exc = e
 
+        # Emit completion or failure event based on result (only for tools with specific streaming events)
+        has_error = error_exc or (result and ((result.error_code and result.error_code > 0) or result.error_message))
+        completion_event = None
+
+        if ctx.mcp_tool_to_server and function.name in ctx.mcp_tool_to_server:
+            sequence_number += 1
+            if has_error:
+                completion_event = OpenAIResponseObjectStreamResponseMcpCallFailed(
+                    sequence_number=sequence_number,
+                )
+            else:
+                completion_event = OpenAIResponseObjectStreamResponseMcpCallCompleted(
+                    sequence_number=sequence_number,
+                )
+        elif function.name == "web_search":
+            sequence_number += 1
+            completion_event = OpenAIResponseObjectStreamResponseWebSearchCallCompleted(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+        # Note: knowledge_search and other custom tools don't have specific completion events in OpenAI spec
+
+        if completion_event:
+            yield ToolExecutionResult(stream_event=completion_event, sequence_number=sequence_number)
+
+        # Build the result message and input message
         if function.name in ctx.mcp_tool_to_server:
             from llama_stack.apis.agents.openai_responses import (
                 OpenAIResponseOutputMessageMCPCall,
@@ -907,9 +1005,9 @@ class OpenAIResponsesImpl:
             )
             if error_exc:
                 message.error = str(error_exc)
-            elif (result.error_code and result.error_code > 0) or result.error_message:
+            elif (result and result.error_code and result.error_code > 0) or (result and result.error_message):
                 message.error = f"Error (code {result.error_code}): {result.error_message}"
-            elif result.content:
+            elif result and result.content:
                 message.output = interleaved_content_as_str(result.content)
         else:
             if function.name == "web_search":
@@ -917,7 +1015,7 @@ class OpenAIResponsesImpl:
                     id=tool_call_id,
                     status="completed",
                 )
-                if error_exc or (result.error_code and result.error_code > 0) or result.error_message:
+                if has_error:
                     message.status = "failed"
             elif function.name == "knowledge_search":
                 message = OpenAIResponseOutputMessageFileSearchToolCall(
@@ -925,7 +1023,7 @@ class OpenAIResponsesImpl:
                     queries=[tool_kwargs.get("query", "")],
                     status="completed",
                 )
-                if "document_ids" in result.metadata:
+                if result and "document_ids" in result.metadata:
                     message.results = []
                     for i, doc_id in enumerate(result.metadata["document_ids"]):
                         text = result.metadata["chunks"][i] if "chunks" in result.metadata else None
@@ -939,7 +1037,7 @@ class OpenAIResponsesImpl:
                                 attributes={},
                             )
                         )
-                if error_exc or (result.error_code and result.error_code > 0) or result.error_message:
+                if has_error:
                     message.status = "failed"
             else:
                 raise ValueError(f"Unknown tool {function.name} called")
@@ -971,10 +1069,13 @@ class OpenAIResponsesImpl:
                 raise ValueError(f"Unknown result content type: {type(result.content)}")
             input_message = OpenAIToolMessageParam(content=content, tool_call_id=tool_call_id)
         else:
-            text = str(error_exc)
+            text = str(error_exc) if error_exc else "Tool execution failed"
             input_message = OpenAIToolMessageParam(content=text, tool_call_id=tool_call_id)
 
-        return message, input_message
+        # Yield the final result
+        yield ToolExecutionResult(
+            sequence_number=sequence_number, final_output_message=message, final_input_message=input_message
+        )
 
 
 def _is_function_tool_call(
diff --git a/tests/integration/non_ci/responses/test_responses.py b/tests/integration/non_ci/responses/test_responses.py
index 6092346b0..776e3cf30 100644
--- a/tests/integration/non_ci/responses/test_responses.py
+++ b/tests/integration/non_ci/responses/test_responses.py
@@ -598,6 +598,10 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         item_added_events = [chunk for chunk in chunks if chunk.type == "response.output_item.added"]
         item_done_events = [chunk for chunk in chunks if chunk.type == "response.output_item.done"]
 
+        # Should have tool execution progress events
+        mcp_in_progress_events = [chunk for chunk in chunks if chunk.type == "response.mcp_call.in_progress"]
+        mcp_completed_events = [chunk for chunk in chunks if chunk.type == "response.mcp_call.completed"]
+
         # Verify we have substantial streaming activity (not just batch events)
         assert len(chunks) > 10, f"Expected rich streaming with many events, got only {len(chunks)} chunks"
 
@@ -609,6 +613,24 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         assert len(item_added_events) > 0, f"Expected response.output_item.added events, got chunk types: {chunk_types}"
         assert len(item_done_events) > 0, f"Expected response.output_item.done events, got chunk types: {chunk_types}"
 
+        # Should have tool execution progress events
+        assert len(mcp_in_progress_events) > 0, (
+            f"Expected response.mcp_call.in_progress events, got chunk types: {chunk_types}"
+        )
+        assert len(mcp_completed_events) > 0, (
+            f"Expected response.mcp_call.completed events, got chunk types: {chunk_types}"
+        )
+        # MCP failed events are optional (only if errors occur)
+
+        # Verify progress events have proper structure
+        for progress_event in mcp_in_progress_events:
+            assert hasattr(progress_event, "item_id"), "Progress event should have 'item_id' field"
+            assert hasattr(progress_event, "output_index"), "Progress event should have 'output_index' field"
+            assert hasattr(progress_event, "sequence_number"), "Progress event should have 'sequence_number' field"
+
+        for completed_event in mcp_completed_events:
+            assert hasattr(completed_event, "sequence_number"), "Completed event should have 'sequence_number' field"
+
         # Verify delta events have proper structure
         for delta_event in delta_events:
             assert hasattr(delta_event, "delta"), "Delta event should have 'delta' field"

From e1e161553c323e2477f24c7091a74cb51f18ef78 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Wed, 13 Aug 2025 16:34:26 -0700
Subject: [PATCH 23/45] feat(responses): add MCP argument streaming and content
 part events (#3136)

# What does this PR do?

Adds content part streaming events to the OpenAI-compatible Responses API to support more granular streaming of response content. This introduces:

1. New schema types for content parts: `OpenAIResponseContentPart` with variants for text output and refusals

2. New streaming event types:
   - `OpenAIResponseObjectStreamResponseContentPartAdded` for when content parts begin
   - `OpenAIResponseObjectStreamResponseContentPartDone` for when content parts complete

3. Implementation in the reference provider to emit these events during streaming responses. Also emits MCP arguments just like function call ones.


## Test Plan

Updated existing streaming tests to verify content part events are properly emitted
---
 docs/_static/llama-stack-spec.html            | 137 ++++++++++++++++++
 docs/_static/llama-stack-spec.yaml            | 111 ++++++++++++++
 llama_stack/apis/agents/openai_responses.py   |  58 ++++++++
 .../agents/meta_reference/openai_responses.py |  96 ++++++++++--
 .../non_ci/responses/test_responses.py        |  77 ++++++++--
 .../meta_reference/test_openai_responses.py   |  36 ++++-
 6 files changed, 480 insertions(+), 35 deletions(-)

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index 25f916d87..0549dda21 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -8821,6 +8821,61 @@
                 "title": "OpenAIResponseOutputMessageMCPListTools",
                 "description": "MCP list tools output message containing available tools from an MCP server."
             },
+            "OpenAIResponseContentPart": {
+                "oneOf": [
+                    {
+                        "$ref": "#/components/schemas/OpenAIResponseContentPartOutputText"
+                    },
+                    {
+                        "$ref": "#/components/schemas/OpenAIResponseContentPartRefusal"
+                    }
+                ],
+                "discriminator": {
+                    "propertyName": "type",
+                    "mapping": {
+                        "output_text": "#/components/schemas/OpenAIResponseContentPartOutputText",
+                        "refusal": "#/components/schemas/OpenAIResponseContentPartRefusal"
+                    }
+                }
+            },
+            "OpenAIResponseContentPartOutputText": {
+                "type": "object",
+                "properties": {
+                    "type": {
+                        "type": "string",
+                        "const": "output_text",
+                        "default": "output_text"
+                    },
+                    "text": {
+                        "type": "string"
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "type",
+                    "text"
+                ],
+                "title": "OpenAIResponseContentPartOutputText"
+            },
+            "OpenAIResponseContentPartRefusal": {
+                "type": "object",
+                "properties": {
+                    "type": {
+                        "type": "string",
+                        "const": "refusal",
+                        "default": "refusal"
+                    },
+                    "refusal": {
+                        "type": "string"
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "type",
+                    "refusal"
+                ],
+                "title": "OpenAIResponseContentPartRefusal"
+            },
             "OpenAIResponseObjectStream": {
                 "oneOf": [
                     {
@@ -8877,6 +8932,12 @@
                     {
                         "$ref": "#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallCompleted"
                     },
+                    {
+                        "$ref": "#/components/schemas/OpenAIResponseObjectStreamResponseContentPartAdded"
+                    },
+                    {
+                        "$ref": "#/components/schemas/OpenAIResponseObjectStreamResponseContentPartDone"
+                    },
                     {
                         "$ref": "#/components/schemas/OpenAIResponseObjectStreamResponseCompleted"
                     }
@@ -8902,6 +8963,8 @@
                         "response.mcp_call.in_progress": "#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallInProgress",
                         "response.mcp_call.failed": "#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallFailed",
                         "response.mcp_call.completed": "#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallCompleted",
+                        "response.content_part.added": "#/components/schemas/OpenAIResponseObjectStreamResponseContentPartAdded",
+                        "response.content_part.done": "#/components/schemas/OpenAIResponseObjectStreamResponseContentPartDone",
                         "response.completed": "#/components/schemas/OpenAIResponseObjectStreamResponseCompleted"
                     }
                 }
@@ -8928,6 +8991,80 @@
                 "title": "OpenAIResponseObjectStreamResponseCompleted",
                 "description": "Streaming event indicating a response has been completed."
             },
+            "OpenAIResponseObjectStreamResponseContentPartAdded": {
+                "type": "object",
+                "properties": {
+                    "response_id": {
+                        "type": "string",
+                        "description": "Unique identifier of the response containing this content"
+                    },
+                    "item_id": {
+                        "type": "string",
+                        "description": "Unique identifier of the output item containing this content part"
+                    },
+                    "part": {
+                        "$ref": "#/components/schemas/OpenAIResponseContentPart",
+                        "description": "The content part that was added"
+                    },
+                    "sequence_number": {
+                        "type": "integer",
+                        "description": "Sequential number for ordering streaming events"
+                    },
+                    "type": {
+                        "type": "string",
+                        "const": "response.content_part.added",
+                        "default": "response.content_part.added",
+                        "description": "Event type identifier, always \"response.content_part.added\""
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "response_id",
+                    "item_id",
+                    "part",
+                    "sequence_number",
+                    "type"
+                ],
+                "title": "OpenAIResponseObjectStreamResponseContentPartAdded",
+                "description": "Streaming event for when a new content part is added to a response item."
+            },
+            "OpenAIResponseObjectStreamResponseContentPartDone": {
+                "type": "object",
+                "properties": {
+                    "response_id": {
+                        "type": "string",
+                        "description": "Unique identifier of the response containing this content"
+                    },
+                    "item_id": {
+                        "type": "string",
+                        "description": "Unique identifier of the output item containing this content part"
+                    },
+                    "part": {
+                        "$ref": "#/components/schemas/OpenAIResponseContentPart",
+                        "description": "The completed content part"
+                    },
+                    "sequence_number": {
+                        "type": "integer",
+                        "description": "Sequential number for ordering streaming events"
+                    },
+                    "type": {
+                        "type": "string",
+                        "const": "response.content_part.done",
+                        "default": "response.content_part.done",
+                        "description": "Event type identifier, always \"response.content_part.done\""
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "response_id",
+                    "item_id",
+                    "part",
+                    "sequence_number",
+                    "type"
+                ],
+                "title": "OpenAIResponseObjectStreamResponseContentPartDone",
+                "description": "Streaming event for when a content part is completed."
+            },
             "OpenAIResponseObjectStreamResponseCreated": {
                 "type": "object",
                 "properties": {
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index 43e9fa95a..aa47cd58d 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -6441,6 +6441,43 @@ components:
       title: OpenAIResponseOutputMessageMCPListTools
       description: >-
         MCP list tools output message containing available tools from an MCP server.
+    OpenAIResponseContentPart:
+      oneOf:
+        - $ref: '#/components/schemas/OpenAIResponseContentPartOutputText'
+        - $ref: '#/components/schemas/OpenAIResponseContentPartRefusal'
+      discriminator:
+        propertyName: type
+        mapping:
+          output_text: '#/components/schemas/OpenAIResponseContentPartOutputText'
+          refusal: '#/components/schemas/OpenAIResponseContentPartRefusal'
+    OpenAIResponseContentPartOutputText:
+      type: object
+      properties:
+        type:
+          type: string
+          const: output_text
+          default: output_text
+        text:
+          type: string
+      additionalProperties: false
+      required:
+        - type
+        - text
+      title: OpenAIResponseContentPartOutputText
+    OpenAIResponseContentPartRefusal:
+      type: object
+      properties:
+        type:
+          type: string
+          const: refusal
+          default: refusal
+        refusal:
+          type: string
+      additionalProperties: false
+      required:
+        - type
+        - refusal
+      title: OpenAIResponseContentPartRefusal
     OpenAIResponseObjectStream:
       oneOf:
         - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseCreated'
@@ -6461,6 +6498,8 @@ components:
         - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallInProgress'
         - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallFailed'
         - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallCompleted'
+        - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseContentPartAdded'
+        - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseContentPartDone'
         - $ref: '#/components/schemas/OpenAIResponseObjectStreamResponseCompleted'
       discriminator:
         propertyName: type
@@ -6483,6 +6522,8 @@ components:
           response.mcp_call.in_progress: '#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallInProgress'
           response.mcp_call.failed: '#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallFailed'
           response.mcp_call.completed: '#/components/schemas/OpenAIResponseObjectStreamResponseMcpCallCompleted'
+          response.content_part.added: '#/components/schemas/OpenAIResponseObjectStreamResponseContentPartAdded'
+          response.content_part.done: '#/components/schemas/OpenAIResponseObjectStreamResponseContentPartDone'
           response.completed: '#/components/schemas/OpenAIResponseObjectStreamResponseCompleted'
     "OpenAIResponseObjectStreamResponseCompleted":
       type: object
@@ -6504,6 +6545,76 @@ components:
         OpenAIResponseObjectStreamResponseCompleted
       description: >-
         Streaming event indicating a response has been completed.
+    "OpenAIResponseObjectStreamResponseContentPartAdded":
+      type: object
+      properties:
+        response_id:
+          type: string
+          description: >-
+            Unique identifier of the response containing this content
+        item_id:
+          type: string
+          description: >-
+            Unique identifier of the output item containing this content part
+        part:
+          $ref: '#/components/schemas/OpenAIResponseContentPart'
+          description: The content part that was added
+        sequence_number:
+          type: integer
+          description: >-
+            Sequential number for ordering streaming events
+        type:
+          type: string
+          const: response.content_part.added
+          default: response.content_part.added
+          description: >-
+            Event type identifier, always "response.content_part.added"
+      additionalProperties: false
+      required:
+        - response_id
+        - item_id
+        - part
+        - sequence_number
+        - type
+      title: >-
+        OpenAIResponseObjectStreamResponseContentPartAdded
+      description: >-
+        Streaming event for when a new content part is added to a response item.
+    "OpenAIResponseObjectStreamResponseContentPartDone":
+      type: object
+      properties:
+        response_id:
+          type: string
+          description: >-
+            Unique identifier of the response containing this content
+        item_id:
+          type: string
+          description: >-
+            Unique identifier of the output item containing this content part
+        part:
+          $ref: '#/components/schemas/OpenAIResponseContentPart'
+          description: The completed content part
+        sequence_number:
+          type: integer
+          description: >-
+            Sequential number for ordering streaming events
+        type:
+          type: string
+          const: response.content_part.done
+          default: response.content_part.done
+          description: >-
+            Event type identifier, always "response.content_part.done"
+      additionalProperties: false
+      required:
+        - response_id
+        - item_id
+        - part
+        - sequence_number
+        - type
+      title: >-
+        OpenAIResponseObjectStreamResponseContentPartDone
+      description: >-
+        Streaming event for when a content part is completed.
     "OpenAIResponseObjectStreamResponseCreated":
       type: object
       properties:
diff --git a/llama_stack/apis/agents/openai_responses.py b/llama_stack/apis/agents/openai_responses.py
index 8574104dc..591992479 100644
--- a/llama_stack/apis/agents/openai_responses.py
+++ b/llama_stack/apis/agents/openai_responses.py
@@ -623,6 +623,62 @@ class OpenAIResponseObjectStreamResponseMcpCallCompleted(BaseModel):
     type: Literal["response.mcp_call.completed"] = "response.mcp_call.completed"
 
 
+@json_schema_type
+class OpenAIResponseContentPartOutputText(BaseModel):
+    type: Literal["output_text"] = "output_text"
+    text: str
+    # TODO: add annotations, logprobs, etc.
+
+
+@json_schema_type
+class OpenAIResponseContentPartRefusal(BaseModel):
+    type: Literal["refusal"] = "refusal"
+    refusal: str
+
+
+OpenAIResponseContentPart = Annotated[
+    OpenAIResponseContentPartOutputText | OpenAIResponseContentPartRefusal,
+    Field(discriminator="type"),
+]
+register_schema(OpenAIResponseContentPart, name="OpenAIResponseContentPart")
+
+
+@json_schema_type
+class OpenAIResponseObjectStreamResponseContentPartAdded(BaseModel):
+    """Streaming event for when a new content part is added to a response item.
+
+    :param response_id: Unique identifier of the response containing this content
+    :param item_id: Unique identifier of the output item containing this content part
+    :param part: The content part that was added
+    :param sequence_number: Sequential number for ordering streaming events
+    :param type: Event type identifier, always "response.content_part.added"
+    """
+
+    response_id: str
+    item_id: str
+    part: OpenAIResponseContentPart
+    sequence_number: int
+    type: Literal["response.content_part.added"] = "response.content_part.added"
+
+
+@json_schema_type
+class OpenAIResponseObjectStreamResponseContentPartDone(BaseModel):
+    """Streaming event for when a content part is completed.
+
+    :param response_id: Unique identifier of the response containing this content
+    :param item_id: Unique identifier of the output item containing this content part
+    :param part: The completed content part
+    :param sequence_number: Sequential number for ordering streaming events
+    :param type: Event type identifier, always "response.content_part.done"
+    """
+
+    response_id: str
+    item_id: str
+    part: OpenAIResponseContentPart
+    sequence_number: int
+    type: Literal["response.content_part.done"] = "response.content_part.done"
+
+
 OpenAIResponseObjectStream = Annotated[
     OpenAIResponseObjectStreamResponseCreated
     | OpenAIResponseObjectStreamResponseOutputItemAdded
@@ -642,6 +698,8 @@ OpenAIResponseObjectStream = Annotated[
     | OpenAIResponseObjectStreamResponseMcpCallInProgress
     | OpenAIResponseObjectStreamResponseMcpCallFailed
     | OpenAIResponseObjectStreamResponseMcpCallCompleted
+    | OpenAIResponseObjectStreamResponseContentPartAdded
+    | OpenAIResponseObjectStreamResponseContentPartDone
     | OpenAIResponseObjectStreamResponseCompleted,
     Field(discriminator="type"),
 ]
diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
index fbb5a608a..6aca4d68e 100644
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
@@ -20,6 +20,7 @@ from llama_stack.apis.agents.openai_responses import (
     ListOpenAIResponseInputItem,
     ListOpenAIResponseObject,
     OpenAIDeleteResponseObject,
+    OpenAIResponseContentPartOutputText,
     OpenAIResponseInput,
     OpenAIResponseInputFunctionToolCallOutput,
     OpenAIResponseInputMessageContent,
@@ -32,9 +33,13 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseObject,
     OpenAIResponseObjectStream,
     OpenAIResponseObjectStreamResponseCompleted,
+    OpenAIResponseObjectStreamResponseContentPartAdded,
+    OpenAIResponseObjectStreamResponseContentPartDone,
     OpenAIResponseObjectStreamResponseCreated,
     OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta,
     OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone,
+    OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta,
+    OpenAIResponseObjectStreamResponseMcpCallArgumentsDone,
     OpenAIResponseObjectStreamResponseMcpCallCompleted,
     OpenAIResponseObjectStreamResponseMcpCallFailed,
     OpenAIResponseObjectStreamResponseMcpCallInProgress,
@@ -475,6 +480,8 @@ class OpenAIResponsesImpl:
             message_item_id = f"msg_{uuid.uuid4()}"
             # Track tool call items for streaming events
             tool_call_item_ids: dict[int, str] = {}
+            # Track content parts for streaming events
+            content_part_emitted = False
 
             async for chunk in completion_result:
                 chat_response_id = chunk.id
@@ -483,6 +490,18 @@ class OpenAIResponsesImpl:
                 for chunk_choice in chunk.choices:
                     # Emit incremental text content as delta events
                     if chunk_choice.delta.content:
+                        # Emit content_part.added event for first text chunk
+                        if not content_part_emitted:
+                            content_part_emitted = True
+                            sequence_number += 1
+                            yield OpenAIResponseObjectStreamResponseContentPartAdded(
+                                response_id=response_id,
+                                item_id=message_item_id,
+                                part=OpenAIResponseContentPartOutputText(
+                                    text="",  # Will be filled incrementally via text deltas
+                                ),
+                                sequence_number=sequence_number,
+                            )
                         sequence_number += 1
                         yield OpenAIResponseObjectStreamResponseOutputTextDelta(
                             content_index=0,
@@ -529,16 +548,33 @@ class OpenAIResponsesImpl:
                                     sequence_number=sequence_number,
                                 )
 
-                            # Stream function call arguments as they arrive
+                            # Stream tool call arguments as they arrive (differentiate between MCP and function calls)
                             if tool_call.function and tool_call.function.arguments:
                                 tool_call_item_id = tool_call_item_ids[tool_call.index]
                                 sequence_number += 1
-                                yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta(
-                                    delta=tool_call.function.arguments,
-                                    item_id=tool_call_item_id,
-                                    output_index=len(output_messages),
-                                    sequence_number=sequence_number,
+
+                                # Check if this is an MCP tool call
+                                is_mcp_tool = (
+                                    ctx.mcp_tool_to_server
+                                    and tool_call.function.name
+                                    and tool_call.function.name in ctx.mcp_tool_to_server
                                 )
+                                if is_mcp_tool:
+                                    # Emit MCP-specific argument delta event
+                                    yield OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta(
+                                        delta=tool_call.function.arguments,
+                                        item_id=tool_call_item_id,
+                                        output_index=len(output_messages),
+                                        sequence_number=sequence_number,
+                                    )
+                                else:
+                                    # Emit function call argument delta event
+                                    yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta(
+                                        delta=tool_call.function.arguments,
+                                        item_id=tool_call_item_id,
+                                        output_index=len(output_messages),
+                                        sequence_number=sequence_number,
+                                    )
 
                                 # Accumulate arguments for final response (only for subsequent chunks)
                                 if not is_new_tool_call:
@@ -546,27 +582,55 @@ class OpenAIResponsesImpl:
                                         response_tool_call.function.arguments or ""
                                     ) + tool_call.function.arguments
 
-            # Emit function_call_arguments.done events for completed tool calls
+            # Emit arguments.done events for completed tool calls (differentiate between MCP and function calls)
             for tool_call_index in sorted(chat_response_tool_calls.keys()):
                 tool_call_item_id = tool_call_item_ids[tool_call_index]
                 final_arguments = chat_response_tool_calls[tool_call_index].function.arguments or ""
+                tool_call_name = chat_response_tool_calls[tool_call_index].function.name
+
+                # Check if this is an MCP tool call
+                is_mcp_tool = ctx.mcp_tool_to_server and tool_call_name and tool_call_name in ctx.mcp_tool_to_server
                 sequence_number += 1
-                yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone(
-                    arguments=final_arguments,
-                    item_id=tool_call_item_id,
-                    output_index=len(output_messages),
-                    sequence_number=sequence_number,
-                )
+                if is_mcp_tool:
+                    # Emit MCP-specific argument done event
+                    yield OpenAIResponseObjectStreamResponseMcpCallArgumentsDone(
+                        arguments=final_arguments,
+                        item_id=tool_call_item_id,
+                        output_index=len(output_messages),
+                        sequence_number=sequence_number,
+                    )
+                else:
+                    # Emit function call argument done event
+                    yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone(
+                        arguments=final_arguments,
+                        item_id=tool_call_item_id,
+                        output_index=len(output_messages),
+                        sequence_number=sequence_number,
+                    )
 
             # Convert collected chunks to complete response
             if chat_response_tool_calls:
                 tool_calls = [chat_response_tool_calls[i] for i in sorted(chat_response_tool_calls.keys())]
-
-                # when there are tool calls, we need to clear the content
-                chat_response_content = []
             else:
                 tool_calls = None
 
+            # Emit content_part.done event if text content was streamed (before content gets cleared)
+            if content_part_emitted:
+                final_text = "".join(chat_response_content)
+                sequence_number += 1
+                yield OpenAIResponseObjectStreamResponseContentPartDone(
+                    response_id=response_id,
+                    item_id=message_item_id,
+                    part=OpenAIResponseContentPartOutputText(
+                        text=final_text,
+                    ),
+                    sequence_number=sequence_number,
+                )
+
+            # Clear content when there are tool calls (OpenAI spec behavior)
+            if chat_response_tool_calls:
+                chat_response_content = []
+
             assistant_message = OpenAIAssistantMessageParam(
                 content="".join(chat_response_content),
                 tool_calls=tool_calls,
diff --git a/tests/integration/non_ci/responses/test_responses.py b/tests/integration/non_ci/responses/test_responses.py
index 776e3cf30..04266eec8 100644
--- a/tests/integration/non_ci/responses/test_responses.py
+++ b/tests/integration/non_ci/responses/test_responses.py
@@ -590,9 +590,17 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         # Verify tool call streaming events are present
         chunk_types = [chunk.type for chunk in chunks]
 
-        # Should have function call arguments delta events for tool calls
-        delta_events = [chunk for chunk in chunks if chunk.type == "response.function_call_arguments.delta"]
-        done_events = [chunk for chunk in chunks if chunk.type == "response.function_call_arguments.done"]
+        # Should have function call or MCP arguments delta/done events for tool calls
+        delta_events = [
+            chunk
+            for chunk in chunks
+            if chunk.type in ["response.function_call_arguments.delta", "response.mcp_call.arguments.delta"]
+        ]
+        done_events = [
+            chunk
+            for chunk in chunks
+            if chunk.type in ["response.function_call_arguments.done", "response.mcp_call.arguments.done"]
+        ]
 
         # Should have output item events for tool calls
         item_added_events = [chunk for chunk in chunks if chunk.type == "response.output_item.added"]
@@ -606,8 +614,12 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         assert len(chunks) > 10, f"Expected rich streaming with many events, got only {len(chunks)} chunks"
 
         # Since this test involves MCP tool calls, we should see streaming events
-        assert len(delta_events) > 0, f"Expected function_call_arguments.delta events, got chunk types: {chunk_types}"
-        assert len(done_events) > 0, f"Expected function_call_arguments.done events, got chunk types: {chunk_types}"
+        assert len(delta_events) > 0, (
+            f"Expected function_call_arguments.delta or mcp_call.arguments.delta events, got chunk types: {chunk_types}"
+        )
+        assert len(done_events) > 0, (
+            f"Expected function_call_arguments.done or mcp_call.arguments.done events, got chunk types: {chunk_types}"
+        )
 
         # Should have output item events for function calls
         assert len(item_added_events) > 0, f"Expected response.output_item.added events, got chunk types: {chunk_types}"
@@ -670,22 +682,32 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
             assert isinstance(done_event.output_index, int), "Output index should be integer"
             assert done_event.output_index >= 0, "Output index should be non-negative"
 
-        # Group function call argument events by item_id (these should have proper tracking)
-        function_call_events_by_item_id = {}
+        # Group function call and MCP argument events by item_id (these should have proper tracking)
+        argument_events_by_item_id = {}
         for chunk in chunks:
             if hasattr(chunk, "item_id") and chunk.type in [
                 "response.function_call_arguments.delta",
                 "response.function_call_arguments.done",
+                "response.mcp_call.arguments.delta",
+                "response.mcp_call.arguments.done",
             ]:
                 item_id = chunk.item_id
-                if item_id not in function_call_events_by_item_id:
-                    function_call_events_by_item_id[item_id] = []
-                function_call_events_by_item_id[item_id].append(chunk)
+                if item_id not in argument_events_by_item_id:
+                    argument_events_by_item_id[item_id] = []
+                argument_events_by_item_id[item_id].append(chunk)
 
-        for item_id, related_events in function_call_events_by_item_id.items():
-            # Should have at least one delta and one done event for a complete function call
-            delta_events = [e for e in related_events if e.type == "response.function_call_arguments.delta"]
-            done_events = [e for e in related_events if e.type == "response.function_call_arguments.done"]
+        for item_id, related_events in argument_events_by_item_id.items():
+            # Should have at least one delta and one done event for a complete tool call
+            delta_events = [
+                e
+                for e in related_events
+                if e.type in ["response.function_call_arguments.delta", "response.mcp_call.arguments.delta"]
+            ]
+            done_events = [
+                e
+                for e in related_events
+                if e.type in ["response.function_call_arguments.done", "response.mcp_call.arguments.done"]
+            ]
 
             assert len(delta_events) > 0, f"Item {item_id} should have at least one delta event"
             assert len(done_events) == 1, f"Item {item_id} should have exactly one done event"
@@ -694,6 +716,33 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
             for event in related_events:
                 assert event.item_id == item_id, f"Event should have consistent item_id {item_id}, got {event.item_id}"
 
+        # Verify content part events if they exist (for text streaming)
+        content_part_added_events = [chunk for chunk in chunks if chunk.type == "response.content_part.added"]
+        content_part_done_events = [chunk for chunk in chunks if chunk.type == "response.content_part.done"]
+
+        # Content part events should be paired (if any exist)
+        if len(content_part_added_events) > 0:
+            assert len(content_part_done_events) > 0, (
+                "Should have content_part.done events if content_part.added events exist"
+            )
+
+            # Verify content part event structure
+            for added_event in content_part_added_events:
+                assert hasattr(added_event, "response_id"), "Content part added event should have response_id"
+                assert hasattr(added_event, "item_id"), "Content part added event should have item_id"
+                assert hasattr(added_event, "part"), "Content part added event should have part"
+
+                # TODO: enable this after the client types are updated
+                # assert added_event.part.type == "output_text", "Content part should be an output_text"
+
+            for done_event in content_part_done_events:
+                assert hasattr(done_event, "response_id"), "Content part done event should have response_id"
+                assert hasattr(done_event, "item_id"), "Content part done event should have item_id"
+                assert hasattr(done_event, "part"), "Content part done event should have part"
+
+                # TODO: enable this after the client types are updated
+                # assert len(done_event.part.text) > 0, "Content part should have text when done"
+
         # Basic pairing check: each output_item.added should be followed by some activity
         # (but we can't enforce strict 1:1 pairing due to the complexity of multi-turn scenarios)
         assert len(item_added_events) > 0, "Should have at least one output_item.added event"
diff --git a/tests/unit/providers/agents/meta_reference/test_openai_responses.py b/tests/unit/providers/agents/meta_reference/test_openai_responses.py
index 855a525e9..4132a74a3 100644
--- a/tests/unit/providers/agents/meta_reference/test_openai_responses.py
+++ b/tests/unit/providers/agents/meta_reference/test_openai_responses.py
@@ -136,9 +136,12 @@ async def test_create_openai_response_with_string_input(openai_responses_impl, m
         input=input_text,
         model=model,
         temperature=0.1,
+        stream=True,  # Enable streaming to test content part events
     )
 
-    # Verify
+    # For streaming response, collect all chunks
+    chunks = [chunk async for chunk in result]
+
     mock_inference_api.openai_chat_completion.assert_called_once_with(
         model=model,
         messages=[OpenAIUserMessageParam(role="user", content="What is the capital of Ireland?", name=None)],
@@ -147,11 +150,32 @@ async def test_create_openai_response_with_string_input(openai_responses_impl, m
         stream=True,
         temperature=0.1,
     )
+
+    # Should have content part events for text streaming
+    # Expected: response.created, content_part.added, output_text.delta, content_part.done, response.completed
+    assert len(chunks) >= 4
+    assert chunks[0].type == "response.created"
+
+    # Check for content part events
+    content_part_added_events = [c for c in chunks if c.type == "response.content_part.added"]
+    content_part_done_events = [c for c in chunks if c.type == "response.content_part.done"]
+    text_delta_events = [c for c in chunks if c.type == "response.output_text.delta"]
+
+    assert len(content_part_added_events) >= 1, "Should have content_part.added event for text"
+    assert len(content_part_done_events) >= 1, "Should have content_part.done event for text"
+    assert len(text_delta_events) >= 1, "Should have text delta events"
+
+    # Verify final event is completion
+    assert chunks[-1].type == "response.completed"
+
+    # When streaming, the final response is in the last chunk
+    final_response = chunks[-1].response
+    assert final_response.model == model
+    assert len(final_response.output) == 1
+    assert isinstance(final_response.output[0], OpenAIResponseMessage)
+
     openai_responses_impl.responses_store.store_response_object.assert_called_once()
-    assert result.model == model
-    assert len(result.output) == 1
-    assert isinstance(result.output[0], OpenAIResponseMessage)
-    assert result.output[0].content[0].text == "Dublin"
+    assert final_response.output[0].content[0].text == "Dublin"
 
 
 async def test_create_openai_response_with_string_input_with_tools(openai_responses_impl, mock_inference_api):
@@ -272,6 +296,8 @@ async def test_create_openai_response_with_tool_call_type_none(openai_responses_
 
     # Check that we got the content from our mocked tool execution result
     chunks = [chunk async for chunk in result]
+
+    # Verify event types
     # Should have: response.created, output_item.added, function_call_arguments.delta,
     # function_call_arguments.done, output_item.done, response.completed
     assert len(chunks) == 6

From 46ff302d87562cf266d2a304f7409593ac7bb0ca Mon Sep 17 00:00:00 2001
From: ehhuang <ehhuang@users.noreply.github.com>
Date: Wed, 13 Aug 2025 18:38:34 -0700
Subject: [PATCH 24/45] chore: Remove Trendshift badge from README (#3137)

## Summary
- This links to a scammy looking website with ads.

## Test plan
---
 README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.md b/README.md
index 8db4580a2..4df4a5372 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,5 @@
 # Llama Stack
 
-<a href="https://trendshift.io/repositories/11824" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11824" alt="meta-llama%2Fllama-stack | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
-
------
 [![PyPI version](https://img.shields.io/pypi/v/llama_stack.svg)](https://pypi.org/project/llama_stack/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/llama-stack)](https://pypi.org/project/llama-stack/)
 [![License](https://img.shields.io/pypi/l/llama_stack.svg)](https://github.com/meta-llama/llama-stack/blob/main/LICENSE)

From de692162afe0151ebac69321effd069b194d2754 Mon Sep 17 00:00:00 2001
From: Matthew Farrellee <matt@cs.wisc.edu>
Date: Thu, 14 Aug 2025 08:42:02 -0500
Subject: [PATCH 25/45] feat: add batches API with OpenAI compatibility (#3088)

Add complete batches API implementation with protocol, providers, and
tests:

Core Infrastructure:
- Add batches API protocol using OpenAI Batch types directly
- Add Api.batches enum value and protocol mapping in resolver
- Add OpenAI "batch" file purpose support
- Include proper error handling (ConflictError, ResourceNotFoundError)

Reference Provider:
- Add ReferenceBatchesImpl with full CRUD operations (create, retrieve,
cancel, list)
- Implement background batch processing with configurable concurrency
- Add SQLite KVStore backend for persistence
- Support /v1/chat/completions endpoint with request validation

Comprehensive Test Suite:
- Add unit tests for provider implementation with validation
- Add integration tests for end-to-end batch processing workflows
- Add error handling tests for validation, malformed inputs, and edge
cases

Configuration:
- Add max_concurrent_batches and max_concurrent_requests_per_batch
options
- Add provider documentation with sample configurations

Test with -

```
$ uv run llama stack build --image-type venv --providers inference=YOU_PICK,files=inline::localfs,batches=inline::reference --run &
$ LLAMA_STACK_CONFIG=http://localhost:8321 uv run pytest tests/unit/providers/batches tests/integration/batches --text-model YOU_PICK
```

addresses #3066
---
 docs/_static/llama-stack-spec.html            |   6 +-
 docs/_static/llama-stack-spec.yaml            |   2 +
 docs/source/concepts/apis.md                  |   1 +
 docs/source/providers/agents/index.md         |   9 +
 docs/source/providers/batches/index.md        |  21 +
 .../providers/batches/inline_reference.md     |  23 +
 docs/source/providers/eval/index.md           |   2 +
 docs/source/providers/inference/index.md      |   6 +
 llama_stack/apis/batches/__init__.py          |   9 +
 llama_stack/apis/batches/batches.py           |  89 +++
 llama_stack/apis/common/errors.py             |   6 +
 llama_stack/apis/datatypes.py                 |   2 +
 llama_stack/apis/files/files.py               |   1 +
 llama_stack/core/resolver.py                  |   2 +
 llama_stack/core/server/server.py             |   5 +
 .../providers/inline/batches/__init__.py      |   5 +
 .../inline/batches/reference/__init__.py      |  36 +
 .../inline/batches/reference/batches.py       | 553 +++++++++++++
 .../inline/batches/reference/config.py        |  40 +
 llama_stack/providers/registry/batches.py     |  26 +
 scripts/provider_codegen.py                   |  22 +
 tests/integration/batches/__init__.py         |   5 +
 tests/integration/batches/conftest.py         | 122 +++
 tests/integration/batches/test_batches.py     | 270 +++++++
 .../batches/test_batches_errors.py            | 693 ++++++++++++++++
 .../unit/providers/batches/test_reference.py  | 753 ++++++++++++++++++
 26 files changed, 2707 insertions(+), 2 deletions(-)
 create mode 100644 docs/source/providers/batches/index.md
 create mode 100644 docs/source/providers/batches/inline_reference.md
 create mode 100644 llama_stack/apis/batches/__init__.py
 create mode 100644 llama_stack/apis/batches/batches.py
 create mode 100644 llama_stack/providers/inline/batches/__init__.py
 create mode 100644 llama_stack/providers/inline/batches/reference/__init__.py
 create mode 100644 llama_stack/providers/inline/batches/reference/batches.py
 create mode 100644 llama_stack/providers/inline/batches/reference/config.py
 create mode 100644 llama_stack/providers/registry/batches.py
 create mode 100644 tests/integration/batches/__init__.py
 create mode 100644 tests/integration/batches/conftest.py
 create mode 100644 tests/integration/batches/test_batches.py
 create mode 100644 tests/integration/batches/test_batches_errors.py
 create mode 100644 tests/unit/providers/batches/test_reference.py

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index 0549dda21..b36626719 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -14767,7 +14767,8 @@
             "OpenAIFilePurpose": {
                 "type": "string",
                 "enum": [
-                    "assistants"
+                    "assistants",
+                    "batch"
                 ],
                 "title": "OpenAIFilePurpose",
                 "description": "Valid purpose values for OpenAI Files API."
@@ -14844,7 +14845,8 @@
                     "purpose": {
                         "type": "string",
                         "enum": [
-                            "assistants"
+                            "assistants",
+                            "batch"
                         ],
                         "description": "The intended purpose of the file"
                     }
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index aa47cd58d..e7733b3c3 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -10951,6 +10951,7 @@ components:
       type: string
       enum:
         - assistants
+        - batch
       title: OpenAIFilePurpose
       description: >-
         Valid purpose values for OpenAI Files API.
@@ -11019,6 +11020,7 @@ components:
           type: string
           enum:
             - assistants
+            - batch
           description: The intended purpose of the file
       additionalProperties: false
       required:
diff --git a/docs/source/concepts/apis.md b/docs/source/concepts/apis.md
index 5a10d6498..f8f73a928 100644
--- a/docs/source/concepts/apis.md
+++ b/docs/source/concepts/apis.md
@@ -18,3 +18,4 @@ We are working on adding a few more APIs to complete the application lifecycle.
 - **Batch Inference**: run inference on a dataset of inputs
 - **Batch Agents**: run agents on a dataset of inputs
 - **Synthetic Data Generation**: generate synthetic data for model development
+- **Batches**: OpenAI-compatible batch management for inference
diff --git a/docs/source/providers/agents/index.md b/docs/source/providers/agents/index.md
index 92bf9edc0..a2c48d4b9 100644
--- a/docs/source/providers/agents/index.md
+++ b/docs/source/providers/agents/index.md
@@ -2,6 +2,15 @@
 
 ## Overview
 
+Agents API for creating and interacting with agentic systems.
+
+    Main functionalities provided by this API:
+    - Create agents with specific instructions and ability to use tools.
+    - Interactions with agents are grouped into sessions ("threads"), and each interaction is called a "turn".
+    - Agents can be provided with various tools (see the ToolGroups and ToolRuntime APIs for more details).
+    - Agents can be provided with various shields (see the Safety API for more details).
+    - Agents can also use Memory to retrieve information from knowledge bases. See the RAG Tool and Vector IO APIs for more details.
+
 This section contains documentation for all available providers for the **agents** API.
 
 ## Providers
diff --git a/docs/source/providers/batches/index.md b/docs/source/providers/batches/index.md
new file mode 100644
index 000000000..2a39a626c
--- /dev/null
+++ b/docs/source/providers/batches/index.md
@@ -0,0 +1,21 @@
+# Batches
+
+## Overview
+
+Protocol for batch processing API operations.
+
+    The Batches API enables efficient processing of multiple requests in a single operation,
+    particularly useful for processing large datasets, batch evaluation workflows, and
+    cost-effective inference at scale.
+
+    Note: This API is currently under active development and may undergo changes.
+
+This section contains documentation for all available providers for the **batches** API.
+
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_reference
+```
diff --git a/docs/source/providers/batches/inline_reference.md b/docs/source/providers/batches/inline_reference.md
new file mode 100644
index 000000000..a58e5124d
--- /dev/null
+++ b/docs/source/providers/batches/inline_reference.md
@@ -0,0 +1,23 @@
+# inline::reference
+
+## Description
+
+Reference implementation of batches API with KVStore persistence.
+
+## Configuration
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Configuration for the key-value store backend. |
+| `max_concurrent_batches` | `<class 'int'>` | No | 1 | Maximum number of concurrent batches to process simultaneously. |
+| `max_concurrent_requests_per_batch` | `<class 'int'>` | No | 10 | Maximum number of concurrent requests to process per batch. |
+
+## Sample Configuration
+
+```yaml
+kvstore:
+  type: sqlite
+  db_path: ${env.SQLITE_STORE_DIR:=~/.llama/dummy}/batches.db
+
+```
+
diff --git a/docs/source/providers/eval/index.md b/docs/source/providers/eval/index.md
index d180d256c..a14fada1d 100644
--- a/docs/source/providers/eval/index.md
+++ b/docs/source/providers/eval/index.md
@@ -2,6 +2,8 @@
 
 ## Overview
 
+Llama Stack Evaluation API for running evaluations on model and agent candidates.
+
 This section contains documentation for all available providers for the **eval** API.
 
 ## Providers
diff --git a/docs/source/providers/inference/index.md b/docs/source/providers/inference/index.md
index 38781e5eb..b6d215474 100644
--- a/docs/source/providers/inference/index.md
+++ b/docs/source/providers/inference/index.md
@@ -2,6 +2,12 @@
 
 ## Overview
 
+Llama Stack Inference API for generating completions, chat completions, and embeddings.
+
+    This API provides the raw interface to the underlying models. Two kinds of models are supported:
+    - LLM models: these models generate "raw" and "chat" (conversational) completions.
+    - Embedding models: these models generate embeddings to be used for semantic search.
+
 This section contains documentation for all available providers for the **inference** API.
 
 ## Providers
diff --git a/llama_stack/apis/batches/__init__.py b/llama_stack/apis/batches/__init__.py
new file mode 100644
index 000000000..9ce7d3d75
--- /dev/null
+++ b/llama_stack/apis/batches/__init__.py
@@ -0,0 +1,9 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from .batches import Batches, BatchObject, ListBatchesResponse
+
+__all__ = ["Batches", "BatchObject", "ListBatchesResponse"]
diff --git a/llama_stack/apis/batches/batches.py b/llama_stack/apis/batches/batches.py
new file mode 100644
index 000000000..9297d8597
--- /dev/null
+++ b/llama_stack/apis/batches/batches.py
@@ -0,0 +1,89 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Literal, Protocol, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+from llama_stack.schema_utils import json_schema_type, webmethod
+
+try:
+    from openai.types import Batch as BatchObject
+except ImportError as e:
+    raise ImportError("OpenAI package is required for batches API. Please install it with: pip install openai") from e
+
+
+@json_schema_type
+class ListBatchesResponse(BaseModel):
+    """Response containing a list of batch objects."""
+
+    object: Literal["list"] = "list"
+    data: list[BatchObject] = Field(..., description="List of batch objects")
+    first_id: str | None = Field(default=None, description="ID of the first batch in the list")
+    last_id: str | None = Field(default=None, description="ID of the last batch in the list")
+    has_more: bool = Field(default=False, description="Whether there are more batches available")
+
+
+@runtime_checkable
+class Batches(Protocol):
+    """Protocol for batch processing API operations.
+
+    The Batches API enables efficient processing of multiple requests in a single operation,
+    particularly useful for processing large datasets, batch evaluation workflows, and
+    cost-effective inference at scale.
+
+    Note: This API is currently under active development and may undergo changes.
+    """
+
+    @webmethod(route="/openai/v1/batches", method="POST")
+    async def create_batch(
+        self,
+        input_file_id: str,
+        endpoint: str,
+        completion_window: Literal["24h"],
+        metadata: dict[str, str] | None = None,
+    ) -> BatchObject:
+        """Create a new batch for processing multiple API requests.
+
+        :param input_file_id: The ID of an uploaded file containing requests for the batch.
+        :param endpoint: The endpoint to be used for all requests in the batch.
+        :param completion_window: The time window within which the batch should be processed.
+        :param metadata: Optional metadata for the batch.
+        :returns: The created batch object.
+        """
+        ...
+
+    @webmethod(route="/openai/v1/batches/{batch_id}", method="GET")
+    async def retrieve_batch(self, batch_id: str) -> BatchObject:
+        """Retrieve information about a specific batch.
+
+        :param batch_id: The ID of the batch to retrieve.
+        :returns: The batch object.
+        """
+        ...
+
+    @webmethod(route="/openai/v1/batches/{batch_id}/cancel", method="POST")
+    async def cancel_batch(self, batch_id: str) -> BatchObject:
+        """Cancel a batch that is in progress.
+
+        :param batch_id: The ID of the batch to cancel.
+        :returns: The updated batch object.
+        """
+        ...
+
+    @webmethod(route="/openai/v1/batches", method="GET")
+    async def list_batches(
+        self,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> ListBatchesResponse:
+        """List all batches for the current user.
+
+        :param after: A cursor for pagination; returns batches after this batch ID.
+        :param limit: Number of batches to return (default 20, max 100).
+        :returns: A list of batch objects.
+        """
+        ...
diff --git a/llama_stack/apis/common/errors.py b/llama_stack/apis/common/errors.py
index 6e0fa0b3c..7104d8db6 100644
--- a/llama_stack/apis/common/errors.py
+++ b/llama_stack/apis/common/errors.py
@@ -64,6 +64,12 @@ class SessionNotFoundError(ValueError):
         super().__init__(message)
 
 
+class ConflictError(ValueError):
+    """raised when an operation cannot be performed due to a conflict with the current state"""
+
+    pass
+
+
 class ModelTypeError(TypeError):
     """raised when a model is present but not the correct type"""
 
diff --git a/llama_stack/apis/datatypes.py b/llama_stack/apis/datatypes.py
index cabe46a2f..87fc95917 100644
--- a/llama_stack/apis/datatypes.py
+++ b/llama_stack/apis/datatypes.py
@@ -86,6 +86,7 @@ class Api(Enum, metaclass=DynamicApiMeta):
     :cvar inference: Text generation, chat completions, and embeddings
     :cvar safety: Content moderation and safety shields
     :cvar agents: Agent orchestration and execution
+    :cvar batches: Batch processing for asynchronous API requests
     :cvar vector_io: Vector database operations and queries
     :cvar datasetio: Dataset input/output operations
     :cvar scoring: Model output evaluation and scoring
@@ -108,6 +109,7 @@ class Api(Enum, metaclass=DynamicApiMeta):
     inference = "inference"
     safety = "safety"
     agents = "agents"
+    batches = "batches"
     vector_io = "vector_io"
     datasetio = "datasetio"
     scoring = "scoring"
diff --git a/llama_stack/apis/files/files.py b/llama_stack/apis/files/files.py
index ba8701e23..a1b9dd4dc 100644
--- a/llama_stack/apis/files/files.py
+++ b/llama_stack/apis/files/files.py
@@ -22,6 +22,7 @@ class OpenAIFilePurpose(StrEnum):
     """
 
     ASSISTANTS = "assistants"
+    BATCH = "batch"
     # TODO: Add other purposes as needed
 
 
diff --git a/llama_stack/core/resolver.py b/llama_stack/core/resolver.py
index 70c78fb01..7ac98dac8 100644
--- a/llama_stack/core/resolver.py
+++ b/llama_stack/core/resolver.py
@@ -8,6 +8,7 @@ import inspect
 from typing import Any
 
 from llama_stack.apis.agents import Agents
+from llama_stack.apis.batches import Batches
 from llama_stack.apis.benchmarks import Benchmarks
 from llama_stack.apis.datasetio import DatasetIO
 from llama_stack.apis.datasets import Datasets
@@ -75,6 +76,7 @@ def api_protocol_map(external_apis: dict[Api, ExternalApiSpec] | None = None) ->
         Api.agents: Agents,
         Api.inference: Inference,
         Api.inspect: Inspect,
+        Api.batches: Batches,
         Api.vector_io: VectorIO,
         Api.vector_dbs: VectorDBs,
         Api.models: Models,
diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py
index e9d70fc8d..cbef8ef88 100644
--- a/llama_stack/core/server/server.py
+++ b/llama_stack/core/server/server.py
@@ -32,6 +32,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
 from openai import BadRequestError
 from pydantic import BaseModel, ValidationError
 
+from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
 from llama_stack.apis.common.responses import PaginatedResponse
 from llama_stack.cli.utils import add_config_distro_args, get_config_from_args
 from llama_stack.core.access_control.access_control import AccessDeniedError
@@ -128,6 +129,10 @@ def translate_exception(exc: Exception) -> HTTPException | RequestValidationErro
                 ]
             },
         )
+    elif isinstance(exc, ConflictError):
+        return HTTPException(status_code=409, detail=str(exc))
+    elif isinstance(exc, ResourceNotFoundError):
+        return HTTPException(status_code=404, detail=str(exc))
     elif isinstance(exc, ValueError):
         return HTTPException(status_code=httpx.codes.BAD_REQUEST, detail=f"Invalid value: {str(exc)}")
     elif isinstance(exc, BadRequestError):
diff --git a/llama_stack/providers/inline/batches/__init__.py b/llama_stack/providers/inline/batches/__init__.py
new file mode 100644
index 000000000..756f351d8
--- /dev/null
+++ b/llama_stack/providers/inline/batches/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
diff --git a/llama_stack/providers/inline/batches/reference/__init__.py b/llama_stack/providers/inline/batches/reference/__init__.py
new file mode 100644
index 000000000..a8ae92eb2
--- /dev/null
+++ b/llama_stack/providers/inline/batches/reference/__init__.py
@@ -0,0 +1,36 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any
+
+from llama_stack.apis.files import Files
+from llama_stack.apis.inference import Inference
+from llama_stack.apis.models import Models
+from llama_stack.core.datatypes import AccessRule, Api
+from llama_stack.providers.utils.kvstore import kvstore_impl
+
+from .batches import ReferenceBatchesImpl
+from .config import ReferenceBatchesImplConfig
+
+__all__ = ["ReferenceBatchesImpl", "ReferenceBatchesImplConfig"]
+
+
+async def get_provider_impl(config: ReferenceBatchesImplConfig, deps: dict[Api, Any], policy: list[AccessRule]):
+    kvstore = await kvstore_impl(config.kvstore)
+    inference_api: Inference | None = deps.get(Api.inference)
+    files_api: Files | None = deps.get(Api.files)
+    models_api: Models | None = deps.get(Api.models)
+
+    if inference_api is None:
+        raise ValueError("Inference API is required but not provided in dependencies")
+    if files_api is None:
+        raise ValueError("Files API is required but not provided in dependencies")
+    if models_api is None:
+        raise ValueError("Models API is required but not provided in dependencies")
+
+    impl = ReferenceBatchesImpl(config, inference_api, files_api, models_api, kvstore)
+    await impl.initialize()
+    return impl
diff --git a/llama_stack/providers/inline/batches/reference/batches.py b/llama_stack/providers/inline/batches/reference/batches.py
new file mode 100644
index 000000000..984ef5a90
--- /dev/null
+++ b/llama_stack/providers/inline/batches/reference/batches.py
@@ -0,0 +1,553 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import asyncio
+import itertools
+import json
+import time
+import uuid
+from io import BytesIO
+from typing import Any, Literal
+
+from openai.types.batch import BatchError, Errors
+from pydantic import BaseModel
+
+from llama_stack.apis.batches import Batches, BatchObject, ListBatchesResponse
+from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
+from llama_stack.apis.files import Files, OpenAIFilePurpose
+from llama_stack.apis.inference import Inference
+from llama_stack.apis.models import Models
+from llama_stack.log import get_logger
+from llama_stack.providers.utils.kvstore import KVStore
+
+from .config import ReferenceBatchesImplConfig
+
+BATCH_PREFIX = "batch:"
+
+logger = get_logger(__name__)
+
+
+class AsyncBytesIO:
+    """
+    Async-compatible BytesIO wrapper to allow async file-like operations.
+
+    We use this when uploading files to the Files API, as it expects an
+    async file-like object.
+    """
+
+    def __init__(self, data: bytes):
+        self._buffer = BytesIO(data)
+
+    async def read(self, n=-1):
+        return self._buffer.read(n)
+
+    async def seek(self, pos, whence=0):
+        return self._buffer.seek(pos, whence)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._buffer.close()
+
+    def __getattr__(self, name):
+        return getattr(self._buffer, name)
+
+
+class BatchRequest(BaseModel):
+    line_num: int
+    custom_id: str
+    method: str
+    url: str
+    body: dict[str, Any]
+
+
+class ReferenceBatchesImpl(Batches):
+    """Reference implementation of the Batches API.
+
+    This implementation processes batch files by making individual requests
+    to the inference API and generates output files with results.
+    """
+
+    def __init__(
+        self,
+        config: ReferenceBatchesImplConfig,
+        inference_api: Inference,
+        files_api: Files,
+        models_api: Models,
+        kvstore: KVStore,
+    ) -> None:
+        self.config = config
+        self.kvstore = kvstore
+        self.inference_api = inference_api
+        self.files_api = files_api
+        self.models_api = models_api
+        self._processing_tasks: dict[str, asyncio.Task] = {}
+        self._batch_semaphore = asyncio.Semaphore(config.max_concurrent_batches)
+        self._update_batch_lock = asyncio.Lock()
+
+        # this is to allow tests to disable background processing
+        self.process_batches = True
+
+    async def initialize(self) -> None:
+        # TODO: start background processing of existing tasks
+        pass
+
+    async def shutdown(self) -> None:
+        """Shutdown the batches provider."""
+        if self._processing_tasks:
+            # don't cancel tasks - just let them stop naturally on shutdown
+            # cancelling would mark batches as "cancelled" in the database
+            logger.info(f"Shutdown initiated with {len(self._processing_tasks)} active batch processing tasks")
+
+    # TODO (SECURITY): this currently works w/ configured api keys, not with x-llamastack-provider-data or with user policy restrictions
+    async def create_batch(
+        self,
+        input_file_id: str,
+        endpoint: str,
+        completion_window: Literal["24h"],
+        metadata: dict[str, str] | None = None,
+    ) -> BatchObject:
+        """
+        Create a new batch for processing multiple API requests.
+
+        Error handling by levels -
+         0. Input param handling, results in 40x errors before processing, e.g.
+           - Wrong completion_window
+           - Invalid metadata types
+           - Unknown endpoint
+          -> no batch created
+         1. Errors preventing processing, result in BatchErrors aggregated in process_batch, e.g.
+           - input_file_id missing
+           - invalid json in file
+           - missing custom_id, method, url, body
+           - invalid model
+           - streaming
+          -> batch created, validation sends to failed status
+         2. Processing errors, result in error_file_id entries, e.g.
+           - Any error returned from inference endpoint
+          -> batch created, goes to completed status
+        """
+
+        # TODO: set expiration time for garbage collection
+
+        if endpoint not in ["/v1/chat/completions"]:
+            raise ValueError(
+                f"Invalid endpoint: {endpoint}. Supported values: /v1/chat/completions. Code: invalid_value. Param: endpoint",
+            )
+
+        if completion_window != "24h":
+            raise ValueError(
+                f"Invalid completion_window: {completion_window}. Supported values are: 24h. Code: invalid_value. Param: completion_window",
+            )
+
+        batch_id = f"batch_{uuid.uuid4().hex[:16]}"
+        current_time = int(time.time())
+
+        batch = BatchObject(
+            id=batch_id,
+            object="batch",
+            endpoint=endpoint,
+            input_file_id=input_file_id,
+            completion_window=completion_window,
+            status="validating",
+            created_at=current_time,
+            metadata=metadata,
+        )
+
+        await self.kvstore.set(f"batch:{batch_id}", batch.to_json())
+
+        if self.process_batches:
+            task = asyncio.create_task(self._process_batch(batch_id))
+            self._processing_tasks[batch_id] = task
+
+        return batch
+
+    async def cancel_batch(self, batch_id: str) -> BatchObject:
+        """Cancel a batch that is in progress."""
+        batch = await self.retrieve_batch(batch_id)
+
+        if batch.status in ["cancelled", "cancelling"]:
+            return batch
+
+        if batch.status in ["completed", "failed", "expired"]:
+            raise ConflictError(f"Cannot cancel batch '{batch_id}' with status '{batch.status}'")
+
+        await self._update_batch(batch_id, status="cancelling", cancelling_at=int(time.time()))
+
+        if batch_id in self._processing_tasks:
+            self._processing_tasks[batch_id].cancel()
+            # note: task removal and status="cancelled" handled in finally block of _process_batch
+
+        return await self.retrieve_batch(batch_id)
+
+    async def list_batches(
+        self,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> ListBatchesResponse:
+        """
+        List all batches, eventually only for the current user.
+
+        With no notion of user, we return all batches.
+        """
+        batch_values = await self.kvstore.values_in_range("batch:", "batch:\xff")
+
+        batches = []
+        for batch_data in batch_values:
+            if batch_data:
+                batches.append(BatchObject.model_validate_json(batch_data))
+
+        batches.sort(key=lambda b: b.created_at, reverse=True)
+
+        start_idx = 0
+        if after:
+            for i, batch in enumerate(batches):
+                if batch.id == after:
+                    start_idx = i + 1
+                    break
+
+        page_batches = batches[start_idx : start_idx + limit]
+        has_more = (start_idx + limit) < len(batches)
+
+        first_id = page_batches[0].id if page_batches else None
+        last_id = page_batches[-1].id if page_batches else None
+
+        return ListBatchesResponse(
+            data=page_batches,
+            first_id=first_id,
+            last_id=last_id,
+            has_more=has_more,
+        )
+
+    async def retrieve_batch(self, batch_id: str) -> BatchObject:
+        """Retrieve information about a specific batch."""
+        batch_data = await self.kvstore.get(f"batch:{batch_id}")
+        if not batch_data:
+            raise ResourceNotFoundError(batch_id, "Batch", "batches.list()")
+
+        return BatchObject.model_validate_json(batch_data)
+
+    async def _update_batch(self, batch_id: str, **updates) -> None:
+        """Update batch fields in kvstore."""
+        async with self._update_batch_lock:
+            try:
+                batch = await self.retrieve_batch(batch_id)
+
+                # batch processing is async. once cancelling, only allow "cancelled" status updates
+                if batch.status == "cancelling" and updates.get("status") != "cancelled":
+                    logger.info(
+                        f"Skipping status update for cancelled batch {batch_id}: attempted {updates.get('status')}"
+                    )
+                    return
+
+                if "errors" in updates:
+                    updates["errors"] = updates["errors"].model_dump()
+
+                batch_dict = batch.model_dump()
+                batch_dict.update(updates)
+
+                await self.kvstore.set(f"batch:{batch_id}", json.dumps(batch_dict))
+            except Exception as e:
+                logger.error(f"Failed to update batch {batch_id}: {e}")
+
+    async def _validate_input(self, batch: BatchObject) -> tuple[list[BatchError], list[BatchRequest]]:
+        """
+        Read & validate input, return errors and valid input.
+
+        Validation of
+        - input_file_id existance
+        - valid json
+        - custom_id, method, url, body presence and valid
+        - no streaming
+        """
+        requests: list[BatchRequest] = []
+        errors: list[BatchError] = []
+        try:
+            await self.files_api.openai_retrieve_file(batch.input_file_id)
+        except Exception:
+            errors.append(
+                BatchError(
+                    code="invalid_request",
+                    line=None,
+                    message=f"Cannot find file {batch.input_file_id}.",
+                    param="input_file_id",
+                )
+            )
+            return errors, requests
+
+        # TODO(SECURITY): do something about large files
+        file_content_response = await self.files_api.openai_retrieve_file_content(batch.input_file_id)
+        file_content = file_content_response.body.decode("utf-8")
+        for line_num, line in enumerate(file_content.strip().split("\n"), 1):
+            if line.strip():  # skip empty lines
+                try:
+                    request = json.loads(line)
+
+                    if not isinstance(request, dict):
+                        errors.append(
+                            BatchError(
+                                code="invalid_request",
+                                line=line_num,
+                                message="Each line must be a JSON dictionary object",
+                            )
+                        )
+                        continue
+
+                    valid = True
+
+                    for param, expected_type, type_string in [
+                        ("custom_id", str, "string"),
+                        ("method", str, "string"),
+                        ("url", str, "string"),
+                        ("body", dict, "JSON dictionary object"),
+                    ]:
+                        if param not in request:
+                            errors.append(
+                                BatchError(
+                                    code="missing_required_parameter",
+                                    line=line_num,
+                                    message=f"Missing required parameter: {param}",
+                                    param=param,
+                                )
+                            )
+                            valid = False
+                        elif not isinstance(request[param], expected_type):
+                            param_name = "URL" if param == "url" else param.capitalize()
+                            errors.append(
+                                BatchError(
+                                    code="invalid_request",
+                                    line=line_num,
+                                    message=f"{param_name} must be a {type_string}",
+                                    param=param,
+                                )
+                            )
+                            valid = False
+
+                    if (url := request.get("url")) and isinstance(url, str) and url != batch.endpoint:
+                        errors.append(
+                            BatchError(
+                                code="invalid_url",
+                                line=line_num,
+                                message="URL provided for this request does not match the batch endpoint",
+                                param="url",
+                            )
+                        )
+                        valid = False
+
+                    if (body := request.get("body")) and isinstance(body, dict):
+                        if body.get("stream", False):
+                            errors.append(
+                                BatchError(
+                                    code="streaming_unsupported",
+                                    line=line_num,
+                                    message="Streaming is not supported in batch processing",
+                                    param="body.stream",
+                                )
+                            )
+                            valid = False
+
+                        for param, expected_type, type_string in [
+                            ("model", str, "a string"),
+                            # messages is specific to /v1/chat/completions
+                            # we could skip validating messages here and let inference fail. however,
+                            # that would be a very expensive way to find out messages is wrong.
+                            ("messages", list, "an array"),  # TODO: allow messages to be a string?
+                        ]:
+                            if param not in body:
+                                errors.append(
+                                    BatchError(
+                                        code="invalid_request",
+                                        line=line_num,
+                                        message=f"{param.capitalize()} parameter is required",
+                                        param=f"body.{param}",
+                                    )
+                                )
+                                valid = False
+                            elif not isinstance(body[param], expected_type):
+                                errors.append(
+                                    BatchError(
+                                        code="invalid_request",
+                                        line=line_num,
+                                        message=f"{param.capitalize()} must be {type_string}",
+                                        param=f"body.{param}",
+                                    )
+                                )
+                                valid = False
+
+                        if "model" in body and isinstance(body["model"], str):
+                            try:
+                                await self.models_api.get_model(body["model"])
+                            except Exception:
+                                errors.append(
+                                    BatchError(
+                                        code="model_not_found",
+                                        line=line_num,
+                                        message=f"Model '{body['model']}' does not exist or is not supported",
+                                        param="body.model",
+                                    )
+                                )
+                                valid = False
+
+                    if valid:
+                        assert isinstance(url, str), "URL must be a string"  # for mypy
+                        assert isinstance(body, dict), "Body must be a dictionary"  # for mypy
+                        requests.append(
+                            BatchRequest(
+                                line_num=line_num,
+                                url=url,
+                                method=request["method"],
+                                custom_id=request["custom_id"],
+                                body=body,
+                            ),
+                        )
+                except json.JSONDecodeError:
+                    errors.append(
+                        BatchError(
+                            code="invalid_json_line",
+                            line=line_num,
+                            message="This line is not parseable as valid JSON.",
+                        )
+                    )
+
+        return errors, requests
+
+    async def _process_batch(self, batch_id: str) -> None:
+        """Background task to process a batch of requests."""
+        try:
+            logger.info(f"Starting batch processing for {batch_id}")
+            async with self._batch_semaphore:  # semaphore to limit concurrency
+                logger.info(f"Acquired semaphore for batch {batch_id}")
+                await self._process_batch_impl(batch_id)
+        except asyncio.CancelledError:
+            logger.info(f"Batch processing cancelled for {batch_id}")
+            await self._update_batch(batch_id, status="cancelled", cancelled_at=int(time.time()))
+        except Exception as e:
+            logger.error(f"Batch processing failed for {batch_id}: {e}")
+            await self._update_batch(
+                batch_id,
+                status="failed",
+                failed_at=int(time.time()),
+                errors=Errors(data=[BatchError(code="internal_error", message=str(e))]),
+            )
+        finally:
+            self._processing_tasks.pop(batch_id, None)
+
+    async def _process_batch_impl(self, batch_id: str) -> None:
+        """Implementation of batch processing logic."""
+        errors: list[BatchError] = []
+        batch = await self.retrieve_batch(batch_id)
+
+        errors, requests = await self._validate_input(batch)
+        if errors:
+            await self._update_batch(batch_id, status="failed", failed_at=int(time.time()), errors=Errors(data=errors))
+            logger.info(f"Batch validation failed for {batch_id} with {len(errors)} errors")
+            return
+
+        logger.info(f"Processing {len(requests)} requests for batch {batch_id}")
+
+        total_requests = len(requests)
+        await self._update_batch(
+            batch_id,
+            status="in_progress",
+            request_counts={"total": total_requests, "completed": 0, "failed": 0},
+        )
+
+        error_results = []
+        success_results = []
+        completed_count = 0
+        failed_count = 0
+
+        for chunk in itertools.batched(requests, self.config.max_concurrent_requests_per_batch):
+            # we use a TaskGroup to ensure all process-single-request tasks are canceled when process-batch is cancelled
+            async with asyncio.TaskGroup() as tg:
+                chunk_tasks = [tg.create_task(self._process_single_request(batch_id, request)) for request in chunk]
+
+                chunk_results = await asyncio.gather(*chunk_tasks, return_exceptions=True)
+
+            for result in chunk_results:
+                if isinstance(result, dict) and result.get("error") is not None:  # error response from inference
+                    failed_count += 1
+                    error_results.append(result)
+                elif isinstance(result, dict) and result.get("response") is not None:  # successful inference
+                    completed_count += 1
+                    success_results.append(result)
+                else:  # unexpected result
+                    failed_count += 1
+                    errors.append(BatchError(code="internal_error", message=f"Unexpected result: {result}"))
+
+            await self._update_batch(
+                batch_id,
+                request_counts={"total": total_requests, "completed": completed_count, "failed": failed_count},
+            )
+
+            if errors:
+                await self._update_batch(
+                    batch_id, status="failed", failed_at=int(time.time()), errors=Errors(data=errors)
+                )
+                return
+
+        try:
+            output_file_id = await self._create_output_file(batch_id, success_results, "success")
+            await self._update_batch(batch_id, output_file_id=output_file_id)
+
+            error_file_id = await self._create_output_file(batch_id, error_results, "error")
+            await self._update_batch(batch_id, error_file_id=error_file_id)
+
+            await self._update_batch(batch_id, status="completed", completed_at=int(time.time()))
+
+            logger.info(
+                f"Batch processing completed for {batch_id}: {completed_count} completed, {failed_count} failed"
+            )
+        except Exception as e:
+            # note: errors is empty at this point, so we don't lose anything by ignoring it
+            await self._update_batch(
+                batch_id,
+                status="failed",
+                failed_at=int(time.time()),
+                errors=Errors(data=[BatchError(code="output_failed", message=str(e))]),
+            )
+
+    async def _process_single_request(self, batch_id: str, request: BatchRequest) -> dict:
+        """Process a single request from the batch."""
+        request_id = f"batch_req_{batch_id}_{request.line_num}"
+
+        try:
+            # TODO(SECURITY): review body for security issues
+            chat_response = await self.inference_api.openai_chat_completion(**request.body)
+
+            # this is for mypy, we don't allow streaming so we'll get the right type
+            assert hasattr(chat_response, "model_dump_json"), "Chat response must have model_dump_json method"
+            return {
+                "id": request_id,
+                "custom_id": request.custom_id,
+                "response": {
+                    "status_code": 200,
+                    "request_id": request_id,  # TODO: should this be different?
+                    "body": chat_response.model_dump_json(),
+                },
+            }
+        except Exception as e:
+            logger.info(f"Error processing request {request.custom_id} in batch {batch_id}: {e}")
+            return {
+                "id": request_id,
+                "custom_id": request.custom_id,
+                "error": {"type": "request_failed", "message": str(e)},
+            }
+
+    async def _create_output_file(self, batch_id: str, results: list[dict], file_type: str) -> str:
+        """
+        Create an output file with batch results.
+
+        This function filters results based on the specified file_type
+        and uploads the file to the Files API.
+        """
+        output_lines = [json.dumps(result) for result in results]
+
+        with AsyncBytesIO("\n".join(output_lines).encode("utf-8")) as file_buffer:
+            file_buffer.filename = f"{batch_id}_{file_type}.jsonl"
+            uploaded_file = await self.files_api.openai_upload_file(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
+            return uploaded_file.id
diff --git a/llama_stack/providers/inline/batches/reference/config.py b/llama_stack/providers/inline/batches/reference/config.py
new file mode 100644
index 000000000..d8d06868b
--- /dev/null
+++ b/llama_stack/providers/inline/batches/reference/config.py
@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from pydantic import BaseModel, Field
+
+from llama_stack.providers.utils.kvstore.config import KVStoreConfig, SqliteKVStoreConfig
+
+
+class ReferenceBatchesImplConfig(BaseModel):
+    """Configuration for the Reference Batches implementation."""
+
+    kvstore: KVStoreConfig = Field(
+        description="Configuration for the key-value store backend.",
+    )
+
+    max_concurrent_batches: int = Field(
+        default=1,
+        description="Maximum number of concurrent batches to process simultaneously.",
+        ge=1,
+    )
+
+    max_concurrent_requests_per_batch: int = Field(
+        default=10,
+        description="Maximum number of concurrent requests to process per batch.",
+        ge=1,
+    )
+
+    # TODO: add a max requests per second rate limiter
+
+    @classmethod
+    def sample_run_config(cls, __distro_dir__: str) -> dict:
+        return {
+            "kvstore": SqliteKVStoreConfig.sample_run_config(
+                __distro_dir__=__distro_dir__,
+                db_name="batches.db",
+            ),
+        }
diff --git a/llama_stack/providers/registry/batches.py b/llama_stack/providers/registry/batches.py
new file mode 100644
index 000000000..de7886efb
--- /dev/null
+++ b/llama_stack/providers/registry/batches.py
@@ -0,0 +1,26 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+
+from llama_stack.providers.datatypes import Api, InlineProviderSpec, ProviderSpec
+
+
+def available_providers() -> list[ProviderSpec]:
+    return [
+        InlineProviderSpec(
+            api=Api.batches,
+            provider_type="inline::reference",
+            pip_packages=["openai"],
+            module="llama_stack.providers.inline.batches.reference",
+            config_class="llama_stack.providers.inline.batches.reference.config.ReferenceBatchesImplConfig",
+            api_dependencies=[
+                Api.inference,
+                Api.files,
+                Api.models,
+            ],
+            description="Reference implementation of batches API with KVStore persistence.",
+        ),
+    ]
diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py
index 717677c52..060acfa72 100755
--- a/scripts/provider_codegen.py
+++ b/scripts/provider_codegen.py
@@ -18,6 +18,23 @@ from llama_stack.core.distribution import get_provider_registry
 REPO_ROOT = Path(__file__).parent.parent
 
 
+def get_api_docstring(api_name: str) -> str | None:
+    """Extract docstring from the API protocol class."""
+    try:
+        # Import the API module dynamically
+        api_module = __import__(f"llama_stack.apis.{api_name}", fromlist=[api_name.title()])
+
+        # Get the main protocol class (usually capitalized API name)
+        protocol_class_name = api_name.title()
+        if hasattr(api_module, protocol_class_name):
+            protocol_class = getattr(api_module, protocol_class_name)
+            return protocol_class.__doc__
+    except (ImportError, AttributeError):
+        pass
+
+    return None
+
+
 class ChangedPathTracker:
     """Track a list of paths we may have changed."""
 
@@ -261,6 +278,11 @@ def process_provider_registry(progress, change_tracker: ChangedPathTracker) -> N
             index_content.append(f"# {api_name.title()}\n")
             index_content.append("## Overview\n")
 
+            api_docstring = get_api_docstring(api_name)
+            if api_docstring:
+                cleaned_docstring = api_docstring.strip()
+                index_content.append(f"{cleaned_docstring}\n")
+
             index_content.append(
                 f"This section contains documentation for all available providers for the **{api_name}** API.\n"
             )
diff --git a/tests/integration/batches/__init__.py b/tests/integration/batches/__init__.py
new file mode 100644
index 000000000..756f351d8
--- /dev/null
+++ b/tests/integration/batches/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
diff --git a/tests/integration/batches/conftest.py b/tests/integration/batches/conftest.py
new file mode 100644
index 000000000..974fe77ab
--- /dev/null
+++ b/tests/integration/batches/conftest.py
@@ -0,0 +1,122 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Shared pytest fixtures for batch tests."""
+
+import json
+import time
+import warnings
+from contextlib import contextmanager
+from io import BytesIO
+
+import pytest
+
+from llama_stack.apis.files import OpenAIFilePurpose
+
+
+class BatchHelper:
+    """Helper class for creating and managing batch input files."""
+
+    def __init__(self, client):
+        """Initialize with either a batch_client or openai_client."""
+        self.client = client
+
+    @contextmanager
+    def create_file(self, content: str | list[dict], filename_prefix="batch_input"):
+        """Context manager for creating and cleaning up batch input files.
+
+        Args:
+            content: Either a list of batch request dictionaries or raw string content
+            filename_prefix: Prefix for the generated filename (or full filename if content is string)
+
+        Yields:
+            The uploaded file object
+        """
+        if isinstance(content, str):
+            # Handle raw string content (e.g., malformed JSONL, empty files)
+            file_content = content.encode("utf-8")
+        else:
+            # Handle list of batch request dictionaries
+            jsonl_content = "\n".join(json.dumps(req) for req in content)
+            file_content = jsonl_content.encode("utf-8")
+
+        filename = filename_prefix if filename_prefix.endswith(".jsonl") else f"{filename_prefix}.jsonl"
+
+        with BytesIO(file_content) as file_buffer:
+            file_buffer.name = filename
+            uploaded_file = self.client.files.create(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
+
+        try:
+            yield uploaded_file
+        finally:
+            try:
+                self.client.files.delete(uploaded_file.id)
+            except Exception:
+                warnings.warn(
+                    f"Failed to cleanup file {uploaded_file.id}: {uploaded_file.filename}",
+                    stacklevel=2,
+                )
+
+    def wait_for(
+        self,
+        batch_id: str,
+        max_wait_time: int = 60,
+        sleep_interval: int | None = None,
+        expected_statuses: set[str] | None = None,
+        timeout_action: str = "fail",
+    ):
+        """Wait for a batch to reach a terminal status.
+
+        Args:
+            batch_id: The batch ID to monitor
+            max_wait_time: Maximum time to wait in seconds (default: 60 seconds)
+            sleep_interval: Time to sleep between checks in seconds (default: 1/10th of max_wait_time, min 1s, max 15s)
+            expected_statuses: Set of expected terminal statuses (default: {"completed"})
+            timeout_action: Action on timeout - "fail" (pytest.fail) or "skip" (pytest.skip)
+
+        Returns:
+            The final batch object
+
+        Raises:
+            pytest.Failed: If batch reaches an unexpected status or timeout_action is "fail"
+            pytest.Skipped: If timeout_action is "skip" on timeout or unexpected status
+        """
+        if sleep_interval is None:
+            # Default to 1/10th of max_wait_time, with min 1s and max 15s
+            sleep_interval = max(1, min(15, max_wait_time // 10))
+
+        if expected_statuses is None:
+            expected_statuses = {"completed"}
+
+        terminal_statuses = {"completed", "failed", "cancelled", "expired"}
+        unexpected_statuses = terminal_statuses - expected_statuses
+
+        start_time = time.time()
+        while time.time() - start_time < max_wait_time:
+            current_batch = self.client.batches.retrieve(batch_id)
+
+            if current_batch.status in expected_statuses:
+                return current_batch
+            elif current_batch.status in unexpected_statuses:
+                error_msg = f"Batch reached unexpected status: {current_batch.status}"
+                if timeout_action == "skip":
+                    pytest.skip(error_msg)
+                else:
+                    pytest.fail(error_msg)
+
+            time.sleep(sleep_interval)
+
+        timeout_msg = f"Batch did not reach expected status {expected_statuses} within {max_wait_time} seconds"
+        if timeout_action == "skip":
+            pytest.skip(timeout_msg)
+        else:
+            pytest.fail(timeout_msg)
+
+
+@pytest.fixture
+def batch_helper(openai_client):
+    """Fixture that provides a BatchHelper instance for OpenAI client."""
+    return BatchHelper(openai_client)
diff --git a/tests/integration/batches/test_batches.py b/tests/integration/batches/test_batches.py
new file mode 100644
index 000000000..1ef3202d0
--- /dev/null
+++ b/tests/integration/batches/test_batches.py
@@ -0,0 +1,270 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Integration tests for the Llama Stack batch processing functionality.
+
+This module contains comprehensive integration tests for the batch processing API,
+using the OpenAI-compatible client interface for consistency.
+
+Test Categories:
+    1. Core Batch Operations:
+        - test_batch_creation_and_retrieval: Comprehensive batch creation, structure validation, and retrieval
+        - test_batch_listing: Basic batch listing functionality
+        - test_batch_immediate_cancellation: Batch cancellation workflow
+        # TODO: cancel during processing
+
+    2. End-to-End Processing:
+        - test_batch_e2e_chat_completions: Full chat completions workflow with output and error validation
+
+Note: Error conditions and edge cases are primarily tested in test_batches_errors.py
+for better organization and separation of concerns.
+
+CLEANUP WARNING: These tests currently create batches that are not automatically
+cleaned up after test completion. This may lead to resource accumulation over
+multiple test runs. Only test_batch_immediate_cancellation properly cancels its batch.
+The test_batch_e2e_chat_completions test does clean up its output and error files.
+"""
+
+import json
+
+
+class TestBatchesIntegration:
+    """Integration tests for the batches API."""
+
+    def test_batch_creation_and_retrieval(self, openai_client, batch_helper, text_model_id):
+        """Test comprehensive batch creation and retrieval scenarios."""
+        test_metadata = {
+            "test_type": "comprehensive",
+            "purpose": "creation_and_retrieval_test",
+            "version": "1.0",
+            "tags": "test,batch",
+        }
+
+        batch_requests = [
+            {
+                "custom_id": "request-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "batch_creation_test") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata=test_metadata,
+            )
+
+            assert batch.endpoint == "/v1/chat/completions"
+            assert batch.input_file_id == uploaded_file.id
+            assert batch.completion_window == "24h"
+            assert batch.metadata == test_metadata
+
+            retrieved_batch = openai_client.batches.retrieve(batch.id)
+
+            assert retrieved_batch.id == batch.id
+            assert retrieved_batch.object == batch.object
+            assert retrieved_batch.endpoint == batch.endpoint
+            assert retrieved_batch.input_file_id == batch.input_file_id
+            assert retrieved_batch.completion_window == batch.completion_window
+            assert retrieved_batch.metadata == batch.metadata
+
+    def test_batch_listing(self, openai_client, batch_helper, text_model_id):
+        """
+        Test batch listing.
+
+        This test creates multiple batches and verifies that they can be listed.
+        It also deletes the input files before execution, which means the batches
+        will appear as failed due to missing input files. This is expected and
+        a good thing, because it means no inference is performed.
+        """
+        batch_ids = []
+
+        for i in range(2):
+            batch_requests = [
+                {
+                    "custom_id": f"request-{i}",
+                    "method": "POST",
+                    "url": "/v1/chat/completions",
+                    "body": {
+                        "model": text_model_id,
+                        "messages": [{"role": "user", "content": f"Hello {i}"}],
+                        "max_tokens": 10,
+                    },
+                }
+            ]
+
+            with batch_helper.create_file(batch_requests, f"batch_input_{i}") as uploaded_file:
+                batch = openai_client.batches.create(
+                    input_file_id=uploaded_file.id,
+                    endpoint="/v1/chat/completions",
+                    completion_window="24h",
+                )
+                batch_ids.append(batch.id)
+
+        batch_list = openai_client.batches.list()
+
+        assert isinstance(batch_list.data, list)
+
+        listed_batch_ids = {b.id for b in batch_list.data}
+        for batch_id in batch_ids:
+            assert batch_id in listed_batch_ids
+
+    def test_batch_immediate_cancellation(self, openai_client, batch_helper, text_model_id):
+        """Test immediate batch cancellation."""
+        batch_requests = [
+            {
+                "custom_id": "request-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            # hopefully cancel the batch before it completes
+            cancelling_batch = openai_client.batches.cancel(batch.id)
+            assert cancelling_batch.status in ["cancelling", "cancelled"]
+            assert isinstance(cancelling_batch.cancelling_at, int), (
+                f"cancelling_at should be int, got {type(cancelling_batch.cancelling_at)}"
+            )
+
+            final_batch = batch_helper.wait_for(
+                batch.id,
+                max_wait_time=3 * 60,  # often takes 10-11 minutes, give it 3 min
+                expected_statuses={"cancelled"},
+                timeout_action="skip",
+            )
+
+        assert final_batch.status == "cancelled"
+        assert isinstance(final_batch.cancelled_at, int), (
+            f"cancelled_at should be int, got {type(final_batch.cancelled_at)}"
+        )
+
+    def test_batch_e2e_chat_completions(self, openai_client, batch_helper, text_model_id):
+        """Test end-to-end batch processing for chat completions with both successful and failed operations."""
+        batch_requests = [
+            {
+                "custom_id": "success-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Say hello"}],
+                    "max_tokens": 20,
+                },
+            },
+            {
+                "custom_id": "error-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "This should fail"}],
+                    "max_tokens": -1,  # Invalid negative max_tokens will cause inference error
+                },
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata={"test": "e2e_success_and_errors_test"},
+            )
+
+            final_batch = batch_helper.wait_for(
+                batch.id,
+                max_wait_time=3 * 60,  # often takes 2-3 minutes
+                expected_statuses={"completed"},
+                timeout_action="skip",
+            )
+
+        # Expecting a completed batch with both successful and failed requests
+        #  Batch(id='batch_xxx',
+        #        completion_window='24h',
+        #        created_at=...,
+        #        endpoint='/v1/chat/completions',
+        #        input_file_id='file-xxx',
+        #        object='batch',
+        #        status='completed',
+        #        output_file_id='file-xxx',
+        #        error_file_id='file-xxx',
+        #        request_counts=BatchRequestCounts(completed=1, failed=1, total=2))
+
+        assert final_batch.status == "completed"
+        assert final_batch.request_counts is not None
+        assert final_batch.request_counts.total == 2
+        assert final_batch.request_counts.completed == 1
+        assert final_batch.request_counts.failed == 1
+
+        assert final_batch.output_file_id is not None, "Output file should exist for successful requests"
+
+        output_content = openai_client.files.content(final_batch.output_file_id)
+        if isinstance(output_content, str):
+            output_text = output_content
+        else:
+            output_text = output_content.content.decode("utf-8")
+
+        output_lines = output_text.strip().split("\n")
+
+        for line in output_lines:
+            result = json.loads(line)
+
+            assert "id" in result
+            assert "custom_id" in result
+            assert result["custom_id"] == "success-1"
+
+            assert "response" in result
+
+            assert result["response"]["status_code"] == 200
+            assert "body" in result["response"]
+            assert "choices" in result["response"]["body"]
+
+        assert final_batch.error_file_id is not None, "Error file should exist for failed requests"
+
+        error_content = openai_client.files.content(final_batch.error_file_id)
+        if isinstance(error_content, str):
+            error_text = error_content
+        else:
+            error_text = error_content.content.decode("utf-8")
+
+        error_lines = error_text.strip().split("\n")
+
+        for line in error_lines:
+            result = json.loads(line)
+
+            assert "id" in result
+            assert "custom_id" in result
+            assert result["custom_id"] == "error-1"
+            assert "error" in result
+            error = result["error"]
+            assert error is not None
+            assert "code" in error or "message" in error, "Error should have code or message"
+
+        deleted_output_file = openai_client.files.delete(final_batch.output_file_id)
+        assert deleted_output_file.deleted, f"Output file {final_batch.output_file_id} was not deleted successfully"
+
+        deleted_error_file = openai_client.files.delete(final_batch.error_file_id)
+        assert deleted_error_file.deleted, f"Error file {final_batch.error_file_id} was not deleted successfully"
diff --git a/tests/integration/batches/test_batches_errors.py b/tests/integration/batches/test_batches_errors.py
new file mode 100644
index 000000000..bc94a182e
--- /dev/null
+++ b/tests/integration/batches/test_batches_errors.py
@@ -0,0 +1,693 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Error handling and edge case tests for the Llama Stack batch processing functionality.
+
+This module focuses exclusively on testing error conditions, validation failures,
+and edge cases for batch operations to ensure robust error handling and graceful
+degradation.
+
+Test Categories:
+    1. File and Input Validation:
+        - test_batch_nonexistent_file_id: Handling invalid file IDs
+        - test_batch_malformed_jsonl: Processing malformed JSONL input files
+        - test_file_malformed_batch_file: Handling malformed files at upload time
+        - test_batch_missing_required_fields: Validation of required request fields
+
+    2. API Endpoint and Model Validation:
+        - test_batch_invalid_endpoint: Invalid endpoint handling during creation
+        - test_batch_error_handling_invalid_model: Error handling with nonexistent models
+        - test_batch_endpoint_mismatch: Validation of endpoint/URL consistency
+
+    3. Batch Lifecycle Error Handling:
+        - test_batch_retrieve_nonexistent: Retrieving non-existent batches
+        - test_batch_cancel_nonexistent: Cancelling non-existent batches
+        - test_batch_cancel_completed: Attempting to cancel completed batches
+
+    4. Parameter and Configuration Validation:
+        - test_batch_invalid_completion_window: Invalid completion window values
+        - test_batch_invalid_metadata_types: Invalid metadata type validation
+        - test_batch_missing_required_body_fields: Validation of required fields in request body
+
+    5. Feature Restriction and Compatibility:
+        - test_batch_streaming_not_supported: Streaming request rejection
+        - test_batch_mixed_streaming_requests: Mixed streaming/non-streaming validation
+
+Note: Core functionality and OpenAI compatibility tests are located in
+test_batches_integration.py for better organization and separation of concerns.
+
+CLEANUP WARNING: These tests create batches to test error conditions but do not
+automatically clean them up after test completion. While most error tests create
+batches that fail quickly, some may create valid batches that consume resources.
+"""
+
+import pytest
+from openai import BadRequestError, ConflictError, NotFoundError
+
+
+class TestBatchesErrorHandling:
+    """Error handling and edge case tests for the batches API using OpenAI client."""
+
+    def test_batch_nonexistent_file_id(self, openai_client, batch_helper):
+        """Test batch creation with nonexistent input file ID."""
+
+        batch = openai_client.batches.create(
+            input_file_id="file-nonexistent-xyz",
+            endpoint="/v1/chat/completions",
+            completion_window="24h",
+        )
+
+        final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='invalid_request',
+        #        line=None,
+        #        message='Cannot find file ..., or organization ... does not have access to it.',
+        #        param='file_id')
+        #    ], object='list'),
+        #    failed_at=1754566971,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.code == "invalid_request"
+        assert "cannot find file" in error.message.lower()
+
+    def test_batch_invalid_endpoint(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with invalid endpoint."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-endpoint",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            with pytest.raises(BadRequestError) as exc_info:
+                openai_client.batches.create(
+                    input_file_id=uploaded_file.id,
+                    endpoint="/v1/invalid/endpoint",
+                    completion_window="24h",
+                )
+
+            # Expected -
+            #  Error code: 400 - {
+            #    'error': {
+            #      'message': "Invalid value: '/v1/invalid/endpoint'. Supported values are: '/v1/chat/completions', '/v1/completions', '/v1/embeddings', and '/v1/responses'.",
+            #      'type': 'invalid_request_error',
+            #      'param': 'endpoint',
+            #      'code': 'invalid_value'
+            #    }
+            #  }
+
+            error_msg = str(exc_info.value).lower()
+            assert exc_info.value.status_code == 400
+            assert "invalid value" in error_msg
+            assert "/v1/invalid/endpoint" in error_msg
+            assert "supported values" in error_msg
+            assert "endpoint" in error_msg
+            assert "invalid_value" in error_msg
+
+    def test_batch_malformed_jsonl(self, openai_client, batch_helper):
+        """
+        Test batch with malformed JSONL input.
+
+        The /v1/files endpoint requires valid JSONL format, so we provide a well formed line
+        before a malformed line to ensure we get to the /v1/batches validation stage.
+        """
+        with batch_helper.create_file(
+            """{"custom_id": "valid", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "test"}}
+{invalid json here""",
+            "malformed_batch_input.jsonl",
+        ) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      ...,
+        #      BatchError(code='invalid_json_line',
+        #                 line=2,
+        #                 message='This line is not parseable as valid JSON.',
+        #                 param=None)
+        #    ], object='list'),
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) > 0
+        error = final_batch.errors.data[-1]  # get last error because first may be about the "test" model
+        assert error.code == "invalid_json_line"
+        assert error.line == 2
+        assert "not" in error.message.lower()
+        assert "valid json" in error.message.lower()
+
+    @pytest.mark.xfail(reason="Not all file providers validate content")
+    @pytest.mark.parametrize("batch_requests", ["", "{malformed json"], ids=["empty", "malformed"])
+    def test_file_malformed_batch_file(self, openai_client, batch_helper, batch_requests):
+        """Test file upload with malformed content."""
+
+        with pytest.raises(BadRequestError) as exc_info:
+            with batch_helper.create_file(batch_requests, "malformed_batch_input_file.jsonl"):
+                # /v1/files rejects the file, we don't get to batch creation
+                pass
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 400
+        assert "invalid file format" in error_msg
+        assert "jsonl" in error_msg
+
+    def test_batch_retrieve_nonexistent(self, openai_client):
+        """Test retrieving nonexistent batch."""
+        with pytest.raises(NotFoundError) as exc_info:
+            openai_client.batches.retrieve("batch-nonexistent-xyz")
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 404
+        assert "no batch found" in error_msg or "not found" in error_msg
+
+    def test_batch_cancel_nonexistent(self, openai_client):
+        """Test cancelling nonexistent batch."""
+        with pytest.raises(NotFoundError) as exc_info:
+            openai_client.batches.cancel("batch-nonexistent-xyz")
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 404
+        assert "no batch found" in error_msg or "not found" in error_msg
+
+    def test_batch_cancel_completed(self, openai_client, batch_helper, text_model_id):
+        """Test cancelling already completed batch."""
+        batch_requests = [
+            {
+                "custom_id": "cancel-completed",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Quick test"}],
+                    "max_tokens": 5,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "cancel_test_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(
+                batch.id,
+                max_wait_time=3 * 60,  # often take 10-11 min, give it 3 min
+                expected_statuses={"completed"},
+                timeout_action="skip",
+            )
+
+        deleted_file = openai_client.files.delete(final_batch.output_file_id)
+        assert deleted_file.deleted, f"File {final_batch.output_file_id} was not deleted successfully"
+
+        with pytest.raises(ConflictError) as exc_info:
+            openai_client.batches.cancel(batch.id)
+
+        # Expecting -
+        #   Error code: 409 - {
+        #     'error': {
+        #       'message': "Cannot cancel a batch with status 'completed'.",
+        #       'type': 'invalid_request_error',
+        #       'param': None,
+        #       'code': None
+        #     }
+        #   }
+        #
+        # NOTE: Same for "failed", cancelling "cancelled" batches is allowed
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 409
+        assert "cannot cancel" in error_msg
+
+    def test_batch_missing_required_fields(self, openai_client, batch_helper, text_model_id):
+        """Test batch with requests missing required fields."""
+        batch_requests = [
+            {
+                # Missing custom_id
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "No custom_id"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "no-method",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "No method"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "no-url",
+                "method": "POST",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "No URL"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "no-body",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests, "missing_fields_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(
+        #      data=[
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=1,
+        #          message="Missing required parameter: 'custom_id'.",
+        #          param='custom_id'
+        #        ),
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=2,
+        #          message="Missing required parameter: 'method'.",
+        #          param='method'
+        #        ),
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=3,
+        #          message="Missing required parameter: 'url'.",
+        #          param='url'
+        #        ),
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=4,
+        #          message="Missing required parameter: 'body'.",
+        #          param='body'
+        #        )
+        #    ], object='list'),
+        #    failed_at=1754566945,
+        #    ...)
+        #  )
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 4
+        no_custom_id_error = final_batch.errors.data[0]
+        assert no_custom_id_error.code == "missing_required_parameter"
+        assert no_custom_id_error.line == 1
+        assert "missing" in no_custom_id_error.message.lower()
+        assert "custom_id" in no_custom_id_error.message.lower()
+        no_method_error = final_batch.errors.data[1]
+        assert no_method_error.code == "missing_required_parameter"
+        assert no_method_error.line == 2
+        assert "missing" in no_method_error.message.lower()
+        assert "method" in no_method_error.message.lower()
+        no_url_error = final_batch.errors.data[2]
+        assert no_url_error.code == "missing_required_parameter"
+        assert no_url_error.line == 3
+        assert "missing" in no_url_error.message.lower()
+        assert "url" in no_url_error.message.lower()
+        no_body_error = final_batch.errors.data[3]
+        assert no_body_error.code == "missing_required_parameter"
+        assert no_body_error.line == 4
+        assert "missing" in no_body_error.message.lower()
+        assert "body" in no_body_error.message.lower()
+
+    def test_batch_invalid_completion_window(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with invalid completion window."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-completion-window",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            for window in ["1h", "48h", "invalid", ""]:
+                with pytest.raises(BadRequestError) as exc_info:
+                    openai_client.batches.create(
+                        input_file_id=uploaded_file.id,
+                        endpoint="/v1/chat/completions",
+                        completion_window=window,
+                    )
+            assert exc_info.value.status_code == 400
+            error_msg = str(exc_info.value).lower()
+            assert "error" in error_msg
+            assert "completion_window" in error_msg
+
+    def test_batch_streaming_not_supported(self, openai_client, batch_helper, text_model_id):
+        """Test that streaming responses are not supported in batches."""
+        batch_requests = [
+            {
+                "custom_id": "streaming-test",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                    "stream": True,  # Not supported
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "streaming_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #       BatchError(code='streaming_unsupported',
+        #         line=1,
+        #         message='Chat Completions: Streaming is not supported in the Batch API.',
+        #         param='body.stream')
+        #    ], object='list'),
+        #    failed_at=1754566965,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.code == "streaming_unsupported"
+        assert error.line == 1
+        assert "streaming" in error.message.lower()
+        assert "not supported" in error.message.lower()
+        assert error.param == "body.stream"
+        assert final_batch.failed_at is not None
+
+    def test_batch_mixed_streaming_requests(self, openai_client, batch_helper, text_model_id):
+        """
+        Test batch with mixed streaming and non-streaming requests.
+
+        This is distinct from test_batch_streaming_not_supported, which tests a single
+        streaming request, to ensure an otherwise valid batch fails when a single
+        streaming request is included.
+        """
+        batch_requests = [
+            {
+                "custom_id": "valid-non-streaming-request",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello without streaming"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "streaming-request",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello with streaming"}],
+                    "max_tokens": 10,
+                    "stream": True,  # Not supported
+                },
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests, "mixed_streaming_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='streaming_unsupported',
+        #        line=2,
+        #        message='Chat Completions: Streaming is not supported in the Batch API.',
+        #        param='body.stream')
+        #    ], object='list'),
+        #    failed_at=1754574442,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.code == "streaming_unsupported"
+        assert error.line == 2
+        assert "streaming" in error.message.lower()
+        assert "not supported" in error.message.lower()
+        assert error.param == "body.stream"
+        assert final_batch.failed_at is not None
+
+    def test_batch_endpoint_mismatch(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with mismatched endpoint and request URL."""
+        batch_requests = [
+            {
+                "custom_id": "endpoint-mismatch",
+                "method": "POST",
+                "url": "/v1/embeddings",  # Different from batch endpoint
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "endpoint_mismatch_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",  # Different from request URL
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='invalid_url',
+        #        line=1,
+        #        message='The URL provided for this request does not match the batch endpoint.',
+        #        param='url')
+        #    ], object='list'),
+        #    failed_at=1754566972,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.line == 1
+        assert error.code == "invalid_url"
+        assert "does not match" in error.message.lower()
+        assert "endpoint" in error.message.lower()
+        assert final_batch.failed_at is not None
+
+    def test_batch_error_handling_invalid_model(self, openai_client, batch_helper):
+        """Test batch error handling with invalid model."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-model",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": "nonexistent-model-xyz",
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(code='model_not_found',
+        #        line=1,
+        #        message="The provided model 'nonexistent-model-xyz' is not supported by the Batch API.",
+        #        param='body.model')
+        #    ], object='list'),
+        #    failed_at=1754566978,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.line == 1
+        assert error.code == "model_not_found"
+        assert "not supported" in error.message.lower()
+        assert error.param == "body.model"
+        assert final_batch.failed_at is not None
+
+    def test_batch_missing_required_body_fields(self, openai_client, batch_helper, text_model_id):
+        """Test batch with requests missing required fields in body (model and messages)."""
+        batch_requests = [
+            {
+                "custom_id": "missing-model",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    # Missing model field
+                    "messages": [{"role": "user", "content": "Hello without model"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "missing-messages",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    # Missing messages field
+                    "max_tokens": 10,
+                },
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests, "missing_body_fields_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='invalid_request',
+        #        line=1,
+        #        message='Model parameter is required.',
+        #        param='body.model'),
+        #      BatchError(
+        #        code='invalid_request',
+        #        line=2,
+        #        message='Messages parameter is required.',
+        #        param='body.messages')
+        #      ], object='list'),
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 2
+
+        model_error = final_batch.errors.data[0]
+        assert model_error.line == 1
+        assert "model" in model_error.message.lower()
+        assert model_error.param == "body.model"
+
+        messages_error = final_batch.errors.data[1]
+        assert messages_error.line == 2
+        assert "messages" in messages_error.message.lower()
+        assert messages_error.param == "body.messages"
+
+        assert final_batch.failed_at is not None
+
+    def test_batch_invalid_metadata_types(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with invalid metadata types (like lists)."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-metadata-type",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            with pytest.raises(Exception) as exc_info:
+                openai_client.batches.create(
+                    input_file_id=uploaded_file.id,
+                    endpoint="/v1/chat/completions",
+                    completion_window="24h",
+                    metadata={
+                        "tags": ["tag1", "tag2"],  # Invalid type, should be a string
+                    },
+                )
+
+        # Expecting -
+        #  Error code: 400 - {'error':
+        #    {'message': "Invalid type for 'metadata.tags': expected a string,
+        #                 but got an array instead.",
+        #     'type': 'invalid_request_error', 'param': 'metadata.tags',
+        #     'code': 'invalid_type'}}
+
+        error_msg = str(exc_info.value).lower()
+        assert "400" in error_msg
+        assert "tags" in error_msg
+        assert "string" in error_msg
diff --git a/tests/unit/providers/batches/test_reference.py b/tests/unit/providers/batches/test_reference.py
new file mode 100644
index 000000000..9fe0cc710
--- /dev/null
+++ b/tests/unit/providers/batches/test_reference.py
@@ -0,0 +1,753 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Test suite for the reference implementation of the Batches API.
+
+The tests are categorized and outlined below, keep this updated:
+
+- Batch creation with various parameters and validation:
+  * test_create_and_retrieve_batch_success (positive)
+  * test_create_batch_without_metadata (positive)
+  * test_create_batch_completion_window (negative)
+  * test_create_batch_invalid_endpoints (negative)
+  * test_create_batch_invalid_metadata (negative)
+
+- Batch retrieval and error handling for non-existent batches:
+  * test_retrieve_batch_not_found (negative)
+
+- Batch cancellation with proper status transitions:
+  * test_cancel_batch_success (positive)
+  * test_cancel_batch_invalid_statuses (negative)
+  * test_cancel_batch_not_found (negative)
+
+- Batch listing with pagination and filtering:
+  * test_list_batches_empty (positive)
+  * test_list_batches_single_batch (positive)
+  * test_list_batches_multiple_batches (positive)
+  * test_list_batches_with_limit (positive)
+  * test_list_batches_with_pagination (positive)
+  * test_list_batches_invalid_after (negative)
+
+- Data persistence in the underlying key-value store:
+  * test_kvstore_persistence (positive)
+
+- Batch processing concurrency control:
+  * test_max_concurrent_batches (positive)
+
+- Input validation testing (direct _validate_input method tests):
+  * test_validate_input_file_not_found (negative)
+  * test_validate_input_file_exists_empty_content (positive)
+  * test_validate_input_file_mixed_valid_invalid_json (mixed)
+  * test_validate_input_invalid_model (negative)
+  * test_validate_input_url_mismatch (negative)
+  * test_validate_input_multiple_errors_per_request (negative)
+  * test_validate_input_invalid_request_format (negative)
+  * test_validate_input_missing_parameters (parametrized negative - custom_id, method, url, body, model, messages missing validation)
+  * test_validate_input_invalid_parameter_types (parametrized negative - custom_id, url, method, body, model, messages type validation)
+
+The tests use temporary SQLite databases for isolation and mock external
+dependencies like inference, files, and models APIs.
+"""
+
+import json
+import tempfile
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from llama_stack.apis.batches import BatchObject
+from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
+from llama_stack.providers.inline.batches.reference.batches import ReferenceBatchesImpl
+from llama_stack.providers.inline.batches.reference.config import ReferenceBatchesImplConfig
+from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig
+
+
+class TestReferenceBatchesImpl:
+    """Test the reference implementation of the Batches API."""
+
+    @pytest.fixture
+    async def provider(self):
+        """Create a test provider instance with temporary database."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            db_path = Path(tmpdir) / "test_batches.db"
+            kvstore_config = SqliteKVStoreConfig(db_path=str(db_path))
+            config = ReferenceBatchesImplConfig(kvstore=kvstore_config)
+
+            # Create kvstore and mock APIs
+            from unittest.mock import AsyncMock
+
+            from llama_stack.providers.utils.kvstore import kvstore_impl
+
+            kvstore = await kvstore_impl(config.kvstore)
+            mock_inference = AsyncMock()
+            mock_files = AsyncMock()
+            mock_models = AsyncMock()
+
+            provider = ReferenceBatchesImpl(config, mock_inference, mock_files, mock_models, kvstore)
+            await provider.initialize()
+
+            # unit tests should not require background processing
+            provider.process_batches = False
+
+            yield provider
+
+            await provider.shutdown()
+
+    @pytest.fixture
+    def sample_batch_data(self):
+        """Sample batch data for testing."""
+        return {
+            "input_file_id": "file_abc123",
+            "endpoint": "/v1/chat/completions",
+            "completion_window": "24h",
+            "metadata": {"test": "true", "priority": "high"},
+        }
+
+    def _validate_batch_type(self, batch, expected_metadata=None):
+        """
+        Helper function to validate batch object structure and field types.
+
+        Note: This validates the direct BatchObject from the provider, not the
+              client library response which has a different structure.
+
+        Args:
+            batch: The BatchObject instance to validate.
+            expected_metadata: Optional expected metadata dictionary to validate against.
+        """
+        assert isinstance(batch.id, str)
+        assert isinstance(batch.completion_window, str)
+        assert isinstance(batch.created_at, int)
+        assert isinstance(batch.endpoint, str)
+        assert isinstance(batch.input_file_id, str)
+        assert batch.object == "batch"
+        assert batch.status in [
+            "validating",
+            "failed",
+            "in_progress",
+            "finalizing",
+            "completed",
+            "expired",
+            "cancelling",
+            "cancelled",
+        ]
+
+        if expected_metadata is not None:
+            assert batch.metadata == expected_metadata
+
+        timestamp_fields = [
+            "cancelled_at",
+            "cancelling_at",
+            "completed_at",
+            "expired_at",
+            "expires_at",
+            "failed_at",
+            "finalizing_at",
+            "in_progress_at",
+        ]
+        for field in timestamp_fields:
+            field_value = getattr(batch, field, None)
+            if field_value is not None:
+                assert isinstance(field_value, int), f"{field} should be int or None, got {type(field_value)}"
+
+        file_id_fields = ["error_file_id", "output_file_id"]
+        for field in file_id_fields:
+            field_value = getattr(batch, field, None)
+            if field_value is not None:
+                assert isinstance(field_value, str), f"{field} should be str or None, got {type(field_value)}"
+
+        if hasattr(batch, "request_counts") and batch.request_counts is not None:
+            assert isinstance(batch.request_counts.completed, int), (
+                f"request_counts.completed should be int, got {type(batch.request_counts.completed)}"
+            )
+            assert isinstance(batch.request_counts.failed, int), (
+                f"request_counts.failed should be int, got {type(batch.request_counts.failed)}"
+            )
+            assert isinstance(batch.request_counts.total, int), (
+                f"request_counts.total should be int, got {type(batch.request_counts.total)}"
+            )
+
+        if hasattr(batch, "errors") and batch.errors is not None:
+            assert isinstance(batch.errors, dict), f"errors should be object or dict, got {type(batch.errors)}"
+
+            if hasattr(batch.errors, "data") and batch.errors.data is not None:
+                assert isinstance(batch.errors.data, list), (
+                    f"errors.data should be list or None, got {type(batch.errors.data)}"
+                )
+
+                for i, error_item in enumerate(batch.errors.data):
+                    assert isinstance(error_item, dict), (
+                        f"errors.data[{i}] should be object or dict, got {type(error_item)}"
+                    )
+
+                    if hasattr(error_item, "code") and error_item.code is not None:
+                        assert isinstance(error_item.code, str), (
+                            f"errors.data[{i}].code should be str or None, got {type(error_item.code)}"
+                        )
+
+                    if hasattr(error_item, "line") and error_item.line is not None:
+                        assert isinstance(error_item.line, int), (
+                            f"errors.data[{i}].line should be int or None, got {type(error_item.line)}"
+                        )
+
+                    if hasattr(error_item, "message") and error_item.message is not None:
+                        assert isinstance(error_item.message, str), (
+                            f"errors.data[{i}].message should be str or None, got {type(error_item.message)}"
+                        )
+
+                    if hasattr(error_item, "param") and error_item.param is not None:
+                        assert isinstance(error_item.param, str), (
+                            f"errors.data[{i}].param should be str or None, got {type(error_item.param)}"
+                        )
+
+            if hasattr(batch.errors, "object") and batch.errors.object is not None:
+                assert isinstance(batch.errors.object, str), (
+                    f"errors.object should be str or None, got {type(batch.errors.object)}"
+                )
+                assert batch.errors.object == "list", f"errors.object should be 'list', got {batch.errors.object}"
+
+    async def test_create_and_retrieve_batch_success(self, provider, sample_batch_data):
+        """Test successful batch creation and retrieval."""
+        created_batch = await provider.create_batch(**sample_batch_data)
+
+        self._validate_batch_type(created_batch, expected_metadata=sample_batch_data["metadata"])
+
+        assert created_batch.id.startswith("batch_")
+        assert len(created_batch.id) > 13
+        assert created_batch.object == "batch"
+        assert created_batch.endpoint == sample_batch_data["endpoint"]
+        assert created_batch.input_file_id == sample_batch_data["input_file_id"]
+        assert created_batch.completion_window == sample_batch_data["completion_window"]
+        assert created_batch.status == "validating"
+        assert created_batch.metadata == sample_batch_data["metadata"]
+        assert isinstance(created_batch.created_at, int)
+        assert created_batch.created_at > 0
+
+        retrieved_batch = await provider.retrieve_batch(created_batch.id)
+
+        self._validate_batch_type(retrieved_batch, expected_metadata=sample_batch_data["metadata"])
+
+        assert retrieved_batch.id == created_batch.id
+        assert retrieved_batch.input_file_id == created_batch.input_file_id
+        assert retrieved_batch.endpoint == created_batch.endpoint
+        assert retrieved_batch.status == created_batch.status
+        assert retrieved_batch.metadata == created_batch.metadata
+
+    async def test_create_batch_without_metadata(self, provider):
+        """Test batch creation without optional metadata."""
+        batch = await provider.create_batch(
+            input_file_id="file_123", endpoint="/v1/chat/completions", completion_window="24h"
+        )
+
+        assert batch.metadata is None
+
+    async def test_create_batch_completion_window(self, provider):
+        """Test batch creation with invalid completion window."""
+        with pytest.raises(ValueError, match="Invalid completion_window"):
+            await provider.create_batch(
+                input_file_id="file_123", endpoint="/v1/chat/completions", completion_window="now"
+            )
+
+    @pytest.mark.parametrize(
+        "endpoint",
+        [
+            "/v1/embeddings",
+            "/v1/completions",
+            "/v1/invalid/endpoint",
+            "",
+        ],
+    )
+    async def test_create_batch_invalid_endpoints(self, provider, endpoint):
+        """Test batch creation with various invalid endpoints."""
+        with pytest.raises(ValueError, match="Invalid endpoint"):
+            await provider.create_batch(input_file_id="file_123", endpoint=endpoint, completion_window="24h")
+
+    async def test_create_batch_invalid_metadata(self, provider):
+        """Test that batch creation fails with invalid metadata."""
+        with pytest.raises(ValueError, match="should be a valid string"):
+            await provider.create_batch(
+                input_file_id="file_123",
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata={123: "invalid_key"},  # Non-string key
+            )
+
+        with pytest.raises(ValueError, match="should be a valid string"):
+            await provider.create_batch(
+                input_file_id="file_123",
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata={"valid_key": 456},  # Non-string value
+            )
+
+    async def test_retrieve_batch_not_found(self, provider):
+        """Test error when retrieving non-existent batch."""
+        with pytest.raises(ResourceNotFoundError, match=r"Batch 'nonexistent_batch' not found"):
+            await provider.retrieve_batch("nonexistent_batch")
+
+    async def test_cancel_batch_success(self, provider, sample_batch_data):
+        """Test successful batch cancellation."""
+        created_batch = await provider.create_batch(**sample_batch_data)
+        assert created_batch.status == "validating"
+
+        cancelled_batch = await provider.cancel_batch(created_batch.id)
+
+        assert cancelled_batch.id == created_batch.id
+        assert cancelled_batch.status in ["cancelling", "cancelled"]
+        assert isinstance(cancelled_batch.cancelling_at, int)
+        assert cancelled_batch.cancelling_at >= created_batch.created_at
+
+    @pytest.mark.parametrize("status", ["failed", "expired", "completed"])
+    async def test_cancel_batch_invalid_statuses(self, provider, sample_batch_data, status):
+        """Test error when cancelling batch in final states."""
+        provider.process_batches = False
+        created_batch = await provider.create_batch(**sample_batch_data)
+
+        # directly update status in kvstore
+        await provider._update_batch(created_batch.id, status=status)
+
+        with pytest.raises(ConflictError, match=f"Cannot cancel batch '{created_batch.id}' with status '{status}'"):
+            await provider.cancel_batch(created_batch.id)
+
+    async def test_cancel_batch_not_found(self, provider):
+        """Test error when cancelling non-existent batch."""
+        with pytest.raises(ResourceNotFoundError, match=r"Batch 'nonexistent_batch' not found"):
+            await provider.cancel_batch("nonexistent_batch")
+
+    async def test_list_batches_empty(self, provider):
+        """Test listing batches when none exist."""
+        response = await provider.list_batches()
+
+        assert response.object == "list"
+        assert response.data == []
+        assert response.first_id is None
+        assert response.last_id is None
+        assert response.has_more is False
+
+    async def test_list_batches_single_batch(self, provider, sample_batch_data):
+        """Test listing batches with single batch."""
+        created_batch = await provider.create_batch(**sample_batch_data)
+
+        response = await provider.list_batches()
+
+        assert len(response.data) == 1
+        self._validate_batch_type(response.data[0], expected_metadata=sample_batch_data["metadata"])
+        assert response.data[0].id == created_batch.id
+        assert response.first_id == created_batch.id
+        assert response.last_id == created_batch.id
+        assert response.has_more is False
+
+    async def test_list_batches_multiple_batches(self, provider):
+        """Test listing multiple batches."""
+        batches = [
+            await provider.create_batch(
+                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+            for i in range(3)
+        ]
+
+        response = await provider.list_batches()
+
+        assert len(response.data) == 3
+
+        batch_ids = {batch.id for batch in response.data}
+        expected_ids = {batch.id for batch in batches}
+        assert batch_ids == expected_ids
+        assert response.has_more is False
+
+        assert response.first_id in expected_ids
+        assert response.last_id in expected_ids
+
+    async def test_list_batches_with_limit(self, provider):
+        """Test listing batches with limit parameter."""
+        batches = [
+            await provider.create_batch(
+                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+            for i in range(3)
+        ]
+
+        response = await provider.list_batches(limit=2)
+
+        assert len(response.data) == 2
+        assert response.has_more is True
+        assert response.first_id == response.data[0].id
+        assert response.last_id == response.data[1].id
+        batch_ids = {batch.id for batch in response.data}
+        expected_ids = {batch.id for batch in batches}
+        assert batch_ids.issubset(expected_ids)
+
+    async def test_list_batches_with_pagination(self, provider):
+        """Test listing batches with pagination using 'after' parameter."""
+        for i in range(3):
+            await provider.create_batch(
+                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+
+        # Get first page
+        first_page = await provider.list_batches(limit=1)
+        assert len(first_page.data) == 1
+        assert first_page.has_more is True
+
+        # Get second page using 'after'
+        second_page = await provider.list_batches(limit=1, after=first_page.data[0].id)
+        assert len(second_page.data) == 1
+        assert second_page.data[0].id != first_page.data[0].id
+
+        # Verify we got the next batch in order
+        all_batches = await provider.list_batches()
+        expected_second_batch_id = all_batches.data[1].id
+        assert second_page.data[0].id == expected_second_batch_id
+
+    async def test_list_batches_invalid_after(self, provider, sample_batch_data):
+        """Test listing batches with invalid 'after' parameter."""
+        await provider.create_batch(**sample_batch_data)
+
+        response = await provider.list_batches(after="nonexistent_batch")
+
+        # Should return all batches (no filtering when 'after' batch not found)
+        assert len(response.data) == 1
+
+    async def test_kvstore_persistence(self, provider, sample_batch_data):
+        """Test that batches are properly persisted in kvstore."""
+        batch = await provider.create_batch(**sample_batch_data)
+
+        stored_data = await provider.kvstore.get(f"batch:{batch.id}")
+        assert stored_data is not None
+
+        stored_batch_dict = json.loads(stored_data)
+        assert stored_batch_dict["id"] == batch.id
+        assert stored_batch_dict["input_file_id"] == sample_batch_data["input_file_id"]
+
+    async def test_validate_input_file_not_found(self, provider):
+        """Test _validate_input when input file does not exist."""
+        provider.files_api.openai_retrieve_file = AsyncMock(side_effect=Exception("File not found"))
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="nonexistent_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+        assert errors[0].code == "invalid_request"
+        assert errors[0].message == "Cannot find file nonexistent_file."
+        assert errors[0].param == "input_file_id"
+        assert errors[0].line is None
+
+    async def test_validate_input_file_exists_empty_content(self, provider):
+        """Test _validate_input when file exists but is empty."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b""
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="empty_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 0
+        assert len(requests) == 0
+
+    async def test_validate_input_file_mixed_valid_invalid_json(self, provider):
+        """Test _validate_input when file contains valid and invalid JSON lines."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        # Line 1: valid JSON with proper body args, Line 2: invalid JSON
+        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]}}\n{invalid json'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="mixed_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        # Should have 1 JSON parsing error from line 2, and 1 valid request from line 1
+        assert len(errors) == 1
+        assert len(requests) == 1
+
+        assert errors[0].code == "invalid_json_line"
+        assert errors[0].line == 2
+        assert errors[0].message == "This line is not parseable as valid JSON."
+
+        assert requests[0].custom_id == "req-1"
+        assert requests[0].method == "POST"
+        assert requests[0].url == "/v1/chat/completions"
+        assert requests[0].body["model"] == "test-model"
+        assert requests[0].body["messages"] == [{"role": "user", "content": "Hello"}]
+
+    async def test_validate_input_invalid_model(self, provider):
+        """Test _validate_input when file contains request with non-existent model."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "nonexistent-model", "messages": [{"role": "user", "content": "Hello"}]}}'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        provider.models_api.get_model = AsyncMock(side_effect=Exception("Model not found"))
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="invalid_model_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "model_not_found"
+        assert errors[0].line == 1
+        assert errors[0].message == "Model 'nonexistent-model' does not exist or is not supported"
+        assert errors[0].param == "body.model"
+
+    @pytest.mark.parametrize(
+        "param_name,param_path,error_code,error_message",
+        [
+            ("custom_id", "custom_id", "missing_required_parameter", "Missing required parameter: custom_id"),
+            ("method", "method", "missing_required_parameter", "Missing required parameter: method"),
+            ("url", "url", "missing_required_parameter", "Missing required parameter: url"),
+            ("body", "body", "missing_required_parameter", "Missing required parameter: body"),
+            ("model", "body.model", "invalid_request", "Model parameter is required"),
+            ("messages", "body.messages", "invalid_request", "Messages parameter is required"),
+        ],
+    )
+    async def test_validate_input_missing_parameters(self, provider, param_name, param_path, error_code, error_message):
+        """Test _validate_input when file contains request with missing required parameters."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+
+        base_request = {
+            "custom_id": "req-1",
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]},
+        }
+
+        # Remove the specific parameter being tested
+        if "." in param_path:
+            top_level, nested_param = param_path.split(".", 1)
+            del base_request[top_level][nested_param]
+        else:
+            del base_request[param_name]
+
+        mock_response.body = json.dumps(base_request).encode()
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id=f"missing_{param_name}_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == error_code
+        assert errors[0].line == 1
+        assert errors[0].message == error_message
+        assert errors[0].param == param_path
+
+    async def test_validate_input_url_mismatch(self, provider):
+        """Test _validate_input when file contains request with URL that doesn't match batch endpoint."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/embeddings", "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]}}'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",  # This doesn't match the URL in the request
+            input_file_id="url_mismatch_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "invalid_url"
+        assert errors[0].line == 1
+        assert errors[0].message == "URL provided for this request does not match the batch endpoint"
+        assert errors[0].param == "url"
+
+    async def test_validate_input_multiple_errors_per_request(self, provider):
+        """Test _validate_input when a single request has multiple validation errors."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        # Request missing custom_id, has invalid URL, and missing model in body
+        mock_response.body = (
+            b'{"method": "POST", "url": "/v1/embeddings", "body": {"messages": [{"role": "user", "content": "Hello"}]}}'
+        )
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",  # Doesn't match /v1/embeddings in request
+            input_file_id="multiple_errors_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) >= 2  # At least missing custom_id and URL mismatch
+        assert len(requests) == 0
+
+        for error in errors:
+            assert error.line == 1
+
+        error_codes = {error.code for error in errors}
+        assert "missing_required_parameter" in error_codes  # missing custom_id
+        assert "invalid_url" in error_codes  # URL mismatch
+
+    async def test_validate_input_invalid_request_format(self, provider):
+        """Test _validate_input when file contains non-object JSON (array, string, number)."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b'["not", "a", "request", "object"]'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="invalid_format_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "invalid_request"
+        assert errors[0].line == 1
+        assert errors[0].message == "Each line must be a JSON dictionary object"
+
+    @pytest.mark.parametrize(
+        "param_name,param_path,invalid_value,error_message",
+        [
+            ("custom_id", "custom_id", 12345, "Custom_id must be a string"),
+            ("url", "url", 123, "URL must be a string"),
+            ("method", "method", ["POST"], "Method must be a string"),
+            ("body", "body", ["not", "valid"], "Body must be a JSON dictionary object"),
+            ("model", "body.model", 123, "Model must be a string"),
+            ("messages", "body.messages", "invalid messages format", "Messages must be an array"),
+        ],
+    )
+    async def test_validate_input_invalid_parameter_types(
+        self, provider, param_name, param_path, invalid_value, error_message
+    ):
+        """Test _validate_input when file contains request with parameters that have invalid types."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+
+        base_request = {
+            "custom_id": "req-1",
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]},
+        }
+
+        # Override the specific parameter with invalid value
+        if "." in param_path:
+            top_level, nested_param = param_path.split(".", 1)
+            base_request[top_level][nested_param] = invalid_value
+        else:
+            base_request[param_name] = invalid_value
+
+        mock_response.body = json.dumps(base_request).encode()
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id=f"invalid_{param_name}_type_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "invalid_request"
+        assert errors[0].line == 1
+        assert errors[0].message == error_message
+        assert errors[0].param == param_path
+
+    async def test_max_concurrent_batches(self, provider):
+        """Test max_concurrent_batches configuration and concurrency control."""
+        import asyncio
+
+        provider._batch_semaphore = asyncio.Semaphore(2)
+
+        provider.process_batches = True  # enable because we're testing background processing
+
+        active_batches = 0
+
+        async def add_and_wait(batch_id: str):
+            nonlocal active_batches
+            active_batches += 1
+            await asyncio.sleep(float("inf"))
+
+        # the first thing done in _process_batch is to acquire the semaphore, then call _process_batch_impl,
+        # so we can replace _process_batch_impl with our mock to control concurrency
+        provider._process_batch_impl = add_and_wait
+
+        for _ in range(3):
+            await provider.create_batch(
+                input_file_id="file_id", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+
+        await asyncio.sleep(0.042)  # let tasks start
+
+        assert active_batches == 2, f"Expected 2 active batches, got {active_batches}"

From ee7631b6cf23793b3921645b896fef45c10aaea7 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Thu, 14 Aug 2025 10:08:54 -0700
Subject: [PATCH 26/45] Revert "feat: add batches API with OpenAI
 compatibility" (#3149)

Reverts llamastack/llama-stack#3088

The PR broke integration tests.
---
 docs/_static/llama-stack-spec.html            |   6 +-
 docs/_static/llama-stack-spec.yaml            |   2 -
 docs/source/concepts/apis.md                  |   1 -
 docs/source/providers/agents/index.md         |   9 -
 docs/source/providers/batches/index.md        |  21 -
 .../providers/batches/inline_reference.md     |  23 -
 docs/source/providers/eval/index.md           |   2 -
 docs/source/providers/inference/index.md      |   6 -
 llama_stack/apis/batches/__init__.py          |   9 -
 llama_stack/apis/batches/batches.py           |  89 ---
 llama_stack/apis/common/errors.py             |   6 -
 llama_stack/apis/datatypes.py                 |   2 -
 llama_stack/apis/files/files.py               |   1 -
 llama_stack/core/resolver.py                  |   2 -
 llama_stack/core/server/server.py             |   5 -
 .../providers/inline/batches/__init__.py      |   5 -
 .../inline/batches/reference/__init__.py      |  36 -
 .../inline/batches/reference/batches.py       | 553 -------------
 .../inline/batches/reference/config.py        |  40 -
 llama_stack/providers/registry/batches.py     |  26 -
 scripts/provider_codegen.py                   |  22 -
 tests/integration/batches/__init__.py         |   5 -
 tests/integration/batches/conftest.py         | 122 ---
 tests/integration/batches/test_batches.py     | 270 -------
 .../batches/test_batches_errors.py            | 693 ----------------
 .../unit/providers/batches/test_reference.py  | 753 ------------------
 26 files changed, 2 insertions(+), 2707 deletions(-)
 delete mode 100644 docs/source/providers/batches/index.md
 delete mode 100644 docs/source/providers/batches/inline_reference.md
 delete mode 100644 llama_stack/apis/batches/__init__.py
 delete mode 100644 llama_stack/apis/batches/batches.py
 delete mode 100644 llama_stack/providers/inline/batches/__init__.py
 delete mode 100644 llama_stack/providers/inline/batches/reference/__init__.py
 delete mode 100644 llama_stack/providers/inline/batches/reference/batches.py
 delete mode 100644 llama_stack/providers/inline/batches/reference/config.py
 delete mode 100644 llama_stack/providers/registry/batches.py
 delete mode 100644 tests/integration/batches/__init__.py
 delete mode 100644 tests/integration/batches/conftest.py
 delete mode 100644 tests/integration/batches/test_batches.py
 delete mode 100644 tests/integration/batches/test_batches_errors.py
 delete mode 100644 tests/unit/providers/batches/test_reference.py

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index b36626719..0549dda21 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -14767,8 +14767,7 @@
             "OpenAIFilePurpose": {
                 "type": "string",
                 "enum": [
-                    "assistants",
-                    "batch"
+                    "assistants"
                 ],
                 "title": "OpenAIFilePurpose",
                 "description": "Valid purpose values for OpenAI Files API."
@@ -14845,8 +14844,7 @@
                     "purpose": {
                         "type": "string",
                         "enum": [
-                            "assistants",
-                            "batch"
+                            "assistants"
                         ],
                         "description": "The intended purpose of the file"
                     }
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index e7733b3c3..aa47cd58d 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -10951,7 +10951,6 @@ components:
       type: string
       enum:
         - assistants
-        - batch
       title: OpenAIFilePurpose
       description: >-
         Valid purpose values for OpenAI Files API.
@@ -11020,7 +11019,6 @@ components:
           type: string
           enum:
             - assistants
-            - batch
           description: The intended purpose of the file
       additionalProperties: false
       required:
diff --git a/docs/source/concepts/apis.md b/docs/source/concepts/apis.md
index f8f73a928..5a10d6498 100644
--- a/docs/source/concepts/apis.md
+++ b/docs/source/concepts/apis.md
@@ -18,4 +18,3 @@ We are working on adding a few more APIs to complete the application lifecycle.
 - **Batch Inference**: run inference on a dataset of inputs
 - **Batch Agents**: run agents on a dataset of inputs
 - **Synthetic Data Generation**: generate synthetic data for model development
-- **Batches**: OpenAI-compatible batch management for inference
diff --git a/docs/source/providers/agents/index.md b/docs/source/providers/agents/index.md
index a2c48d4b9..92bf9edc0 100644
--- a/docs/source/providers/agents/index.md
+++ b/docs/source/providers/agents/index.md
@@ -2,15 +2,6 @@
 
 ## Overview
 
-Agents API for creating and interacting with agentic systems.
-
-    Main functionalities provided by this API:
-    - Create agents with specific instructions and ability to use tools.
-    - Interactions with agents are grouped into sessions ("threads"), and each interaction is called a "turn".
-    - Agents can be provided with various tools (see the ToolGroups and ToolRuntime APIs for more details).
-    - Agents can be provided with various shields (see the Safety API for more details).
-    - Agents can also use Memory to retrieve information from knowledge bases. See the RAG Tool and Vector IO APIs for more details.
-
 This section contains documentation for all available providers for the **agents** API.
 
 ## Providers
diff --git a/docs/source/providers/batches/index.md b/docs/source/providers/batches/index.md
deleted file mode 100644
index 2a39a626c..000000000
--- a/docs/source/providers/batches/index.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Batches
-
-## Overview
-
-Protocol for batch processing API operations.
-
-    The Batches API enables efficient processing of multiple requests in a single operation,
-    particularly useful for processing large datasets, batch evaluation workflows, and
-    cost-effective inference at scale.
-
-    Note: This API is currently under active development and may undergo changes.
-
-This section contains documentation for all available providers for the **batches** API.
-
-## Providers
-
-```{toctree}
-:maxdepth: 1
-
-inline_reference
-```
diff --git a/docs/source/providers/batches/inline_reference.md b/docs/source/providers/batches/inline_reference.md
deleted file mode 100644
index a58e5124d..000000000
--- a/docs/source/providers/batches/inline_reference.md
+++ /dev/null
@@ -1,23 +0,0 @@
-# inline::reference
-
-## Description
-
-Reference implementation of batches API with KVStore persistence.
-
-## Configuration
-
-| Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
-| `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Configuration for the key-value store backend. |
-| `max_concurrent_batches` | `<class 'int'>` | No | 1 | Maximum number of concurrent batches to process simultaneously. |
-| `max_concurrent_requests_per_batch` | `<class 'int'>` | No | 10 | Maximum number of concurrent requests to process per batch. |
-
-## Sample Configuration
-
-```yaml
-kvstore:
-  type: sqlite
-  db_path: ${env.SQLITE_STORE_DIR:=~/.llama/dummy}/batches.db
-
-```
-
diff --git a/docs/source/providers/eval/index.md b/docs/source/providers/eval/index.md
index a14fada1d..d180d256c 100644
--- a/docs/source/providers/eval/index.md
+++ b/docs/source/providers/eval/index.md
@@ -2,8 +2,6 @@
 
 ## Overview
 
-Llama Stack Evaluation API for running evaluations on model and agent candidates.
-
 This section contains documentation for all available providers for the **eval** API.
 
 ## Providers
diff --git a/docs/source/providers/inference/index.md b/docs/source/providers/inference/index.md
index b6d215474..38781e5eb 100644
--- a/docs/source/providers/inference/index.md
+++ b/docs/source/providers/inference/index.md
@@ -2,12 +2,6 @@
 
 ## Overview
 
-Llama Stack Inference API for generating completions, chat completions, and embeddings.
-
-    This API provides the raw interface to the underlying models. Two kinds of models are supported:
-    - LLM models: these models generate "raw" and "chat" (conversational) completions.
-    - Embedding models: these models generate embeddings to be used for semantic search.
-
 This section contains documentation for all available providers for the **inference** API.
 
 ## Providers
diff --git a/llama_stack/apis/batches/__init__.py b/llama_stack/apis/batches/__init__.py
deleted file mode 100644
index 9ce7d3d75..000000000
--- a/llama_stack/apis/batches/__init__.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from .batches import Batches, BatchObject, ListBatchesResponse
-
-__all__ = ["Batches", "BatchObject", "ListBatchesResponse"]
diff --git a/llama_stack/apis/batches/batches.py b/llama_stack/apis/batches/batches.py
deleted file mode 100644
index 9297d8597..000000000
--- a/llama_stack/apis/batches/batches.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from typing import Literal, Protocol, runtime_checkable
-
-from pydantic import BaseModel, Field
-
-from llama_stack.schema_utils import json_schema_type, webmethod
-
-try:
-    from openai.types import Batch as BatchObject
-except ImportError as e:
-    raise ImportError("OpenAI package is required for batches API. Please install it with: pip install openai") from e
-
-
-@json_schema_type
-class ListBatchesResponse(BaseModel):
-    """Response containing a list of batch objects."""
-
-    object: Literal["list"] = "list"
-    data: list[BatchObject] = Field(..., description="List of batch objects")
-    first_id: str | None = Field(default=None, description="ID of the first batch in the list")
-    last_id: str | None = Field(default=None, description="ID of the last batch in the list")
-    has_more: bool = Field(default=False, description="Whether there are more batches available")
-
-
-@runtime_checkable
-class Batches(Protocol):
-    """Protocol for batch processing API operations.
-
-    The Batches API enables efficient processing of multiple requests in a single operation,
-    particularly useful for processing large datasets, batch evaluation workflows, and
-    cost-effective inference at scale.
-
-    Note: This API is currently under active development and may undergo changes.
-    """
-
-    @webmethod(route="/openai/v1/batches", method="POST")
-    async def create_batch(
-        self,
-        input_file_id: str,
-        endpoint: str,
-        completion_window: Literal["24h"],
-        metadata: dict[str, str] | None = None,
-    ) -> BatchObject:
-        """Create a new batch for processing multiple API requests.
-
-        :param input_file_id: The ID of an uploaded file containing requests for the batch.
-        :param endpoint: The endpoint to be used for all requests in the batch.
-        :param completion_window: The time window within which the batch should be processed.
-        :param metadata: Optional metadata for the batch.
-        :returns: The created batch object.
-        """
-        ...
-
-    @webmethod(route="/openai/v1/batches/{batch_id}", method="GET")
-    async def retrieve_batch(self, batch_id: str) -> BatchObject:
-        """Retrieve information about a specific batch.
-
-        :param batch_id: The ID of the batch to retrieve.
-        :returns: The batch object.
-        """
-        ...
-
-    @webmethod(route="/openai/v1/batches/{batch_id}/cancel", method="POST")
-    async def cancel_batch(self, batch_id: str) -> BatchObject:
-        """Cancel a batch that is in progress.
-
-        :param batch_id: The ID of the batch to cancel.
-        :returns: The updated batch object.
-        """
-        ...
-
-    @webmethod(route="/openai/v1/batches", method="GET")
-    async def list_batches(
-        self,
-        after: str | None = None,
-        limit: int = 20,
-    ) -> ListBatchesResponse:
-        """List all batches for the current user.
-
-        :param after: A cursor for pagination; returns batches after this batch ID.
-        :param limit: Number of batches to return (default 20, max 100).
-        :returns: A list of batch objects.
-        """
-        ...
diff --git a/llama_stack/apis/common/errors.py b/llama_stack/apis/common/errors.py
index 7104d8db6..6e0fa0b3c 100644
--- a/llama_stack/apis/common/errors.py
+++ b/llama_stack/apis/common/errors.py
@@ -64,12 +64,6 @@ class SessionNotFoundError(ValueError):
         super().__init__(message)
 
 
-class ConflictError(ValueError):
-    """raised when an operation cannot be performed due to a conflict with the current state"""
-
-    pass
-
-
 class ModelTypeError(TypeError):
     """raised when a model is present but not the correct type"""
 
diff --git a/llama_stack/apis/datatypes.py b/llama_stack/apis/datatypes.py
index 87fc95917..cabe46a2f 100644
--- a/llama_stack/apis/datatypes.py
+++ b/llama_stack/apis/datatypes.py
@@ -86,7 +86,6 @@ class Api(Enum, metaclass=DynamicApiMeta):
     :cvar inference: Text generation, chat completions, and embeddings
     :cvar safety: Content moderation and safety shields
     :cvar agents: Agent orchestration and execution
-    :cvar batches: Batch processing for asynchronous API requests
     :cvar vector_io: Vector database operations and queries
     :cvar datasetio: Dataset input/output operations
     :cvar scoring: Model output evaluation and scoring
@@ -109,7 +108,6 @@ class Api(Enum, metaclass=DynamicApiMeta):
     inference = "inference"
     safety = "safety"
     agents = "agents"
-    batches = "batches"
     vector_io = "vector_io"
     datasetio = "datasetio"
     scoring = "scoring"
diff --git a/llama_stack/apis/files/files.py b/llama_stack/apis/files/files.py
index a1b9dd4dc..ba8701e23 100644
--- a/llama_stack/apis/files/files.py
+++ b/llama_stack/apis/files/files.py
@@ -22,7 +22,6 @@ class OpenAIFilePurpose(StrEnum):
     """
 
     ASSISTANTS = "assistants"
-    BATCH = "batch"
     # TODO: Add other purposes as needed
 
 
diff --git a/llama_stack/core/resolver.py b/llama_stack/core/resolver.py
index 7ac98dac8..70c78fb01 100644
--- a/llama_stack/core/resolver.py
+++ b/llama_stack/core/resolver.py
@@ -8,7 +8,6 @@ import inspect
 from typing import Any
 
 from llama_stack.apis.agents import Agents
-from llama_stack.apis.batches import Batches
 from llama_stack.apis.benchmarks import Benchmarks
 from llama_stack.apis.datasetio import DatasetIO
 from llama_stack.apis.datasets import Datasets
@@ -76,7 +75,6 @@ def api_protocol_map(external_apis: dict[Api, ExternalApiSpec] | None = None) ->
         Api.agents: Agents,
         Api.inference: Inference,
         Api.inspect: Inspect,
-        Api.batches: Batches,
         Api.vector_io: VectorIO,
         Api.vector_dbs: VectorDBs,
         Api.models: Models,
diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py
index cbef8ef88..e9d70fc8d 100644
--- a/llama_stack/core/server/server.py
+++ b/llama_stack/core/server/server.py
@@ -32,7 +32,6 @@ from fastapi.responses import JSONResponse, StreamingResponse
 from openai import BadRequestError
 from pydantic import BaseModel, ValidationError
 
-from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
 from llama_stack.apis.common.responses import PaginatedResponse
 from llama_stack.cli.utils import add_config_distro_args, get_config_from_args
 from llama_stack.core.access_control.access_control import AccessDeniedError
@@ -129,10 +128,6 @@ def translate_exception(exc: Exception) -> HTTPException | RequestValidationErro
                 ]
             },
         )
-    elif isinstance(exc, ConflictError):
-        return HTTPException(status_code=409, detail=str(exc))
-    elif isinstance(exc, ResourceNotFoundError):
-        return HTTPException(status_code=404, detail=str(exc))
     elif isinstance(exc, ValueError):
         return HTTPException(status_code=httpx.codes.BAD_REQUEST, detail=f"Invalid value: {str(exc)}")
     elif isinstance(exc, BadRequestError):
diff --git a/llama_stack/providers/inline/batches/__init__.py b/llama_stack/providers/inline/batches/__init__.py
deleted file mode 100644
index 756f351d8..000000000
--- a/llama_stack/providers/inline/batches/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
diff --git a/llama_stack/providers/inline/batches/reference/__init__.py b/llama_stack/providers/inline/batches/reference/__init__.py
deleted file mode 100644
index a8ae92eb2..000000000
--- a/llama_stack/providers/inline/batches/reference/__init__.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from typing import Any
-
-from llama_stack.apis.files import Files
-from llama_stack.apis.inference import Inference
-from llama_stack.apis.models import Models
-from llama_stack.core.datatypes import AccessRule, Api
-from llama_stack.providers.utils.kvstore import kvstore_impl
-
-from .batches import ReferenceBatchesImpl
-from .config import ReferenceBatchesImplConfig
-
-__all__ = ["ReferenceBatchesImpl", "ReferenceBatchesImplConfig"]
-
-
-async def get_provider_impl(config: ReferenceBatchesImplConfig, deps: dict[Api, Any], policy: list[AccessRule]):
-    kvstore = await kvstore_impl(config.kvstore)
-    inference_api: Inference | None = deps.get(Api.inference)
-    files_api: Files | None = deps.get(Api.files)
-    models_api: Models | None = deps.get(Api.models)
-
-    if inference_api is None:
-        raise ValueError("Inference API is required but not provided in dependencies")
-    if files_api is None:
-        raise ValueError("Files API is required but not provided in dependencies")
-    if models_api is None:
-        raise ValueError("Models API is required but not provided in dependencies")
-
-    impl = ReferenceBatchesImpl(config, inference_api, files_api, models_api, kvstore)
-    await impl.initialize()
-    return impl
diff --git a/llama_stack/providers/inline/batches/reference/batches.py b/llama_stack/providers/inline/batches/reference/batches.py
deleted file mode 100644
index 984ef5a90..000000000
--- a/llama_stack/providers/inline/batches/reference/batches.py
+++ /dev/null
@@ -1,553 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-import asyncio
-import itertools
-import json
-import time
-import uuid
-from io import BytesIO
-from typing import Any, Literal
-
-from openai.types.batch import BatchError, Errors
-from pydantic import BaseModel
-
-from llama_stack.apis.batches import Batches, BatchObject, ListBatchesResponse
-from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
-from llama_stack.apis.files import Files, OpenAIFilePurpose
-from llama_stack.apis.inference import Inference
-from llama_stack.apis.models import Models
-from llama_stack.log import get_logger
-from llama_stack.providers.utils.kvstore import KVStore
-
-from .config import ReferenceBatchesImplConfig
-
-BATCH_PREFIX = "batch:"
-
-logger = get_logger(__name__)
-
-
-class AsyncBytesIO:
-    """
-    Async-compatible BytesIO wrapper to allow async file-like operations.
-
-    We use this when uploading files to the Files API, as it expects an
-    async file-like object.
-    """
-
-    def __init__(self, data: bytes):
-        self._buffer = BytesIO(data)
-
-    async def read(self, n=-1):
-        return self._buffer.read(n)
-
-    async def seek(self, pos, whence=0):
-        return self._buffer.seek(pos, whence)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self._buffer.close()
-
-    def __getattr__(self, name):
-        return getattr(self._buffer, name)
-
-
-class BatchRequest(BaseModel):
-    line_num: int
-    custom_id: str
-    method: str
-    url: str
-    body: dict[str, Any]
-
-
-class ReferenceBatchesImpl(Batches):
-    """Reference implementation of the Batches API.
-
-    This implementation processes batch files by making individual requests
-    to the inference API and generates output files with results.
-    """
-
-    def __init__(
-        self,
-        config: ReferenceBatchesImplConfig,
-        inference_api: Inference,
-        files_api: Files,
-        models_api: Models,
-        kvstore: KVStore,
-    ) -> None:
-        self.config = config
-        self.kvstore = kvstore
-        self.inference_api = inference_api
-        self.files_api = files_api
-        self.models_api = models_api
-        self._processing_tasks: dict[str, asyncio.Task] = {}
-        self._batch_semaphore = asyncio.Semaphore(config.max_concurrent_batches)
-        self._update_batch_lock = asyncio.Lock()
-
-        # this is to allow tests to disable background processing
-        self.process_batches = True
-
-    async def initialize(self) -> None:
-        # TODO: start background processing of existing tasks
-        pass
-
-    async def shutdown(self) -> None:
-        """Shutdown the batches provider."""
-        if self._processing_tasks:
-            # don't cancel tasks - just let them stop naturally on shutdown
-            # cancelling would mark batches as "cancelled" in the database
-            logger.info(f"Shutdown initiated with {len(self._processing_tasks)} active batch processing tasks")
-
-    # TODO (SECURITY): this currently works w/ configured api keys, not with x-llamastack-provider-data or with user policy restrictions
-    async def create_batch(
-        self,
-        input_file_id: str,
-        endpoint: str,
-        completion_window: Literal["24h"],
-        metadata: dict[str, str] | None = None,
-    ) -> BatchObject:
-        """
-        Create a new batch for processing multiple API requests.
-
-        Error handling by levels -
-         0. Input param handling, results in 40x errors before processing, e.g.
-           - Wrong completion_window
-           - Invalid metadata types
-           - Unknown endpoint
-          -> no batch created
-         1. Errors preventing processing, result in BatchErrors aggregated in process_batch, e.g.
-           - input_file_id missing
-           - invalid json in file
-           - missing custom_id, method, url, body
-           - invalid model
-           - streaming
-          -> batch created, validation sends to failed status
-         2. Processing errors, result in error_file_id entries, e.g.
-           - Any error returned from inference endpoint
-          -> batch created, goes to completed status
-        """
-
-        # TODO: set expiration time for garbage collection
-
-        if endpoint not in ["/v1/chat/completions"]:
-            raise ValueError(
-                f"Invalid endpoint: {endpoint}. Supported values: /v1/chat/completions. Code: invalid_value. Param: endpoint",
-            )
-
-        if completion_window != "24h":
-            raise ValueError(
-                f"Invalid completion_window: {completion_window}. Supported values are: 24h. Code: invalid_value. Param: completion_window",
-            )
-
-        batch_id = f"batch_{uuid.uuid4().hex[:16]}"
-        current_time = int(time.time())
-
-        batch = BatchObject(
-            id=batch_id,
-            object="batch",
-            endpoint=endpoint,
-            input_file_id=input_file_id,
-            completion_window=completion_window,
-            status="validating",
-            created_at=current_time,
-            metadata=metadata,
-        )
-
-        await self.kvstore.set(f"batch:{batch_id}", batch.to_json())
-
-        if self.process_batches:
-            task = asyncio.create_task(self._process_batch(batch_id))
-            self._processing_tasks[batch_id] = task
-
-        return batch
-
-    async def cancel_batch(self, batch_id: str) -> BatchObject:
-        """Cancel a batch that is in progress."""
-        batch = await self.retrieve_batch(batch_id)
-
-        if batch.status in ["cancelled", "cancelling"]:
-            return batch
-
-        if batch.status in ["completed", "failed", "expired"]:
-            raise ConflictError(f"Cannot cancel batch '{batch_id}' with status '{batch.status}'")
-
-        await self._update_batch(batch_id, status="cancelling", cancelling_at=int(time.time()))
-
-        if batch_id in self._processing_tasks:
-            self._processing_tasks[batch_id].cancel()
-            # note: task removal and status="cancelled" handled in finally block of _process_batch
-
-        return await self.retrieve_batch(batch_id)
-
-    async def list_batches(
-        self,
-        after: str | None = None,
-        limit: int = 20,
-    ) -> ListBatchesResponse:
-        """
-        List all batches, eventually only for the current user.
-
-        With no notion of user, we return all batches.
-        """
-        batch_values = await self.kvstore.values_in_range("batch:", "batch:\xff")
-
-        batches = []
-        for batch_data in batch_values:
-            if batch_data:
-                batches.append(BatchObject.model_validate_json(batch_data))
-
-        batches.sort(key=lambda b: b.created_at, reverse=True)
-
-        start_idx = 0
-        if after:
-            for i, batch in enumerate(batches):
-                if batch.id == after:
-                    start_idx = i + 1
-                    break
-
-        page_batches = batches[start_idx : start_idx + limit]
-        has_more = (start_idx + limit) < len(batches)
-
-        first_id = page_batches[0].id if page_batches else None
-        last_id = page_batches[-1].id if page_batches else None
-
-        return ListBatchesResponse(
-            data=page_batches,
-            first_id=first_id,
-            last_id=last_id,
-            has_more=has_more,
-        )
-
-    async def retrieve_batch(self, batch_id: str) -> BatchObject:
-        """Retrieve information about a specific batch."""
-        batch_data = await self.kvstore.get(f"batch:{batch_id}")
-        if not batch_data:
-            raise ResourceNotFoundError(batch_id, "Batch", "batches.list()")
-
-        return BatchObject.model_validate_json(batch_data)
-
-    async def _update_batch(self, batch_id: str, **updates) -> None:
-        """Update batch fields in kvstore."""
-        async with self._update_batch_lock:
-            try:
-                batch = await self.retrieve_batch(batch_id)
-
-                # batch processing is async. once cancelling, only allow "cancelled" status updates
-                if batch.status == "cancelling" and updates.get("status") != "cancelled":
-                    logger.info(
-                        f"Skipping status update for cancelled batch {batch_id}: attempted {updates.get('status')}"
-                    )
-                    return
-
-                if "errors" in updates:
-                    updates["errors"] = updates["errors"].model_dump()
-
-                batch_dict = batch.model_dump()
-                batch_dict.update(updates)
-
-                await self.kvstore.set(f"batch:{batch_id}", json.dumps(batch_dict))
-            except Exception as e:
-                logger.error(f"Failed to update batch {batch_id}: {e}")
-
-    async def _validate_input(self, batch: BatchObject) -> tuple[list[BatchError], list[BatchRequest]]:
-        """
-        Read & validate input, return errors and valid input.
-
-        Validation of
-        - input_file_id existance
-        - valid json
-        - custom_id, method, url, body presence and valid
-        - no streaming
-        """
-        requests: list[BatchRequest] = []
-        errors: list[BatchError] = []
-        try:
-            await self.files_api.openai_retrieve_file(batch.input_file_id)
-        except Exception:
-            errors.append(
-                BatchError(
-                    code="invalid_request",
-                    line=None,
-                    message=f"Cannot find file {batch.input_file_id}.",
-                    param="input_file_id",
-                )
-            )
-            return errors, requests
-
-        # TODO(SECURITY): do something about large files
-        file_content_response = await self.files_api.openai_retrieve_file_content(batch.input_file_id)
-        file_content = file_content_response.body.decode("utf-8")
-        for line_num, line in enumerate(file_content.strip().split("\n"), 1):
-            if line.strip():  # skip empty lines
-                try:
-                    request = json.loads(line)
-
-                    if not isinstance(request, dict):
-                        errors.append(
-                            BatchError(
-                                code="invalid_request",
-                                line=line_num,
-                                message="Each line must be a JSON dictionary object",
-                            )
-                        )
-                        continue
-
-                    valid = True
-
-                    for param, expected_type, type_string in [
-                        ("custom_id", str, "string"),
-                        ("method", str, "string"),
-                        ("url", str, "string"),
-                        ("body", dict, "JSON dictionary object"),
-                    ]:
-                        if param not in request:
-                            errors.append(
-                                BatchError(
-                                    code="missing_required_parameter",
-                                    line=line_num,
-                                    message=f"Missing required parameter: {param}",
-                                    param=param,
-                                )
-                            )
-                            valid = False
-                        elif not isinstance(request[param], expected_type):
-                            param_name = "URL" if param == "url" else param.capitalize()
-                            errors.append(
-                                BatchError(
-                                    code="invalid_request",
-                                    line=line_num,
-                                    message=f"{param_name} must be a {type_string}",
-                                    param=param,
-                                )
-                            )
-                            valid = False
-
-                    if (url := request.get("url")) and isinstance(url, str) and url != batch.endpoint:
-                        errors.append(
-                            BatchError(
-                                code="invalid_url",
-                                line=line_num,
-                                message="URL provided for this request does not match the batch endpoint",
-                                param="url",
-                            )
-                        )
-                        valid = False
-
-                    if (body := request.get("body")) and isinstance(body, dict):
-                        if body.get("stream", False):
-                            errors.append(
-                                BatchError(
-                                    code="streaming_unsupported",
-                                    line=line_num,
-                                    message="Streaming is not supported in batch processing",
-                                    param="body.stream",
-                                )
-                            )
-                            valid = False
-
-                        for param, expected_type, type_string in [
-                            ("model", str, "a string"),
-                            # messages is specific to /v1/chat/completions
-                            # we could skip validating messages here and let inference fail. however,
-                            # that would be a very expensive way to find out messages is wrong.
-                            ("messages", list, "an array"),  # TODO: allow messages to be a string?
-                        ]:
-                            if param not in body:
-                                errors.append(
-                                    BatchError(
-                                        code="invalid_request",
-                                        line=line_num,
-                                        message=f"{param.capitalize()} parameter is required",
-                                        param=f"body.{param}",
-                                    )
-                                )
-                                valid = False
-                            elif not isinstance(body[param], expected_type):
-                                errors.append(
-                                    BatchError(
-                                        code="invalid_request",
-                                        line=line_num,
-                                        message=f"{param.capitalize()} must be {type_string}",
-                                        param=f"body.{param}",
-                                    )
-                                )
-                                valid = False
-
-                        if "model" in body and isinstance(body["model"], str):
-                            try:
-                                await self.models_api.get_model(body["model"])
-                            except Exception:
-                                errors.append(
-                                    BatchError(
-                                        code="model_not_found",
-                                        line=line_num,
-                                        message=f"Model '{body['model']}' does not exist or is not supported",
-                                        param="body.model",
-                                    )
-                                )
-                                valid = False
-
-                    if valid:
-                        assert isinstance(url, str), "URL must be a string"  # for mypy
-                        assert isinstance(body, dict), "Body must be a dictionary"  # for mypy
-                        requests.append(
-                            BatchRequest(
-                                line_num=line_num,
-                                url=url,
-                                method=request["method"],
-                                custom_id=request["custom_id"],
-                                body=body,
-                            ),
-                        )
-                except json.JSONDecodeError:
-                    errors.append(
-                        BatchError(
-                            code="invalid_json_line",
-                            line=line_num,
-                            message="This line is not parseable as valid JSON.",
-                        )
-                    )
-
-        return errors, requests
-
-    async def _process_batch(self, batch_id: str) -> None:
-        """Background task to process a batch of requests."""
-        try:
-            logger.info(f"Starting batch processing for {batch_id}")
-            async with self._batch_semaphore:  # semaphore to limit concurrency
-                logger.info(f"Acquired semaphore for batch {batch_id}")
-                await self._process_batch_impl(batch_id)
-        except asyncio.CancelledError:
-            logger.info(f"Batch processing cancelled for {batch_id}")
-            await self._update_batch(batch_id, status="cancelled", cancelled_at=int(time.time()))
-        except Exception as e:
-            logger.error(f"Batch processing failed for {batch_id}: {e}")
-            await self._update_batch(
-                batch_id,
-                status="failed",
-                failed_at=int(time.time()),
-                errors=Errors(data=[BatchError(code="internal_error", message=str(e))]),
-            )
-        finally:
-            self._processing_tasks.pop(batch_id, None)
-
-    async def _process_batch_impl(self, batch_id: str) -> None:
-        """Implementation of batch processing logic."""
-        errors: list[BatchError] = []
-        batch = await self.retrieve_batch(batch_id)
-
-        errors, requests = await self._validate_input(batch)
-        if errors:
-            await self._update_batch(batch_id, status="failed", failed_at=int(time.time()), errors=Errors(data=errors))
-            logger.info(f"Batch validation failed for {batch_id} with {len(errors)} errors")
-            return
-
-        logger.info(f"Processing {len(requests)} requests for batch {batch_id}")
-
-        total_requests = len(requests)
-        await self._update_batch(
-            batch_id,
-            status="in_progress",
-            request_counts={"total": total_requests, "completed": 0, "failed": 0},
-        )
-
-        error_results = []
-        success_results = []
-        completed_count = 0
-        failed_count = 0
-
-        for chunk in itertools.batched(requests, self.config.max_concurrent_requests_per_batch):
-            # we use a TaskGroup to ensure all process-single-request tasks are canceled when process-batch is cancelled
-            async with asyncio.TaskGroup() as tg:
-                chunk_tasks = [tg.create_task(self._process_single_request(batch_id, request)) for request in chunk]
-
-                chunk_results = await asyncio.gather(*chunk_tasks, return_exceptions=True)
-
-            for result in chunk_results:
-                if isinstance(result, dict) and result.get("error") is not None:  # error response from inference
-                    failed_count += 1
-                    error_results.append(result)
-                elif isinstance(result, dict) and result.get("response") is not None:  # successful inference
-                    completed_count += 1
-                    success_results.append(result)
-                else:  # unexpected result
-                    failed_count += 1
-                    errors.append(BatchError(code="internal_error", message=f"Unexpected result: {result}"))
-
-            await self._update_batch(
-                batch_id,
-                request_counts={"total": total_requests, "completed": completed_count, "failed": failed_count},
-            )
-
-            if errors:
-                await self._update_batch(
-                    batch_id, status="failed", failed_at=int(time.time()), errors=Errors(data=errors)
-                )
-                return
-
-        try:
-            output_file_id = await self._create_output_file(batch_id, success_results, "success")
-            await self._update_batch(batch_id, output_file_id=output_file_id)
-
-            error_file_id = await self._create_output_file(batch_id, error_results, "error")
-            await self._update_batch(batch_id, error_file_id=error_file_id)
-
-            await self._update_batch(batch_id, status="completed", completed_at=int(time.time()))
-
-            logger.info(
-                f"Batch processing completed for {batch_id}: {completed_count} completed, {failed_count} failed"
-            )
-        except Exception as e:
-            # note: errors is empty at this point, so we don't lose anything by ignoring it
-            await self._update_batch(
-                batch_id,
-                status="failed",
-                failed_at=int(time.time()),
-                errors=Errors(data=[BatchError(code="output_failed", message=str(e))]),
-            )
-
-    async def _process_single_request(self, batch_id: str, request: BatchRequest) -> dict:
-        """Process a single request from the batch."""
-        request_id = f"batch_req_{batch_id}_{request.line_num}"
-
-        try:
-            # TODO(SECURITY): review body for security issues
-            chat_response = await self.inference_api.openai_chat_completion(**request.body)
-
-            # this is for mypy, we don't allow streaming so we'll get the right type
-            assert hasattr(chat_response, "model_dump_json"), "Chat response must have model_dump_json method"
-            return {
-                "id": request_id,
-                "custom_id": request.custom_id,
-                "response": {
-                    "status_code": 200,
-                    "request_id": request_id,  # TODO: should this be different?
-                    "body": chat_response.model_dump_json(),
-                },
-            }
-        except Exception as e:
-            logger.info(f"Error processing request {request.custom_id} in batch {batch_id}: {e}")
-            return {
-                "id": request_id,
-                "custom_id": request.custom_id,
-                "error": {"type": "request_failed", "message": str(e)},
-            }
-
-    async def _create_output_file(self, batch_id: str, results: list[dict], file_type: str) -> str:
-        """
-        Create an output file with batch results.
-
-        This function filters results based on the specified file_type
-        and uploads the file to the Files API.
-        """
-        output_lines = [json.dumps(result) for result in results]
-
-        with AsyncBytesIO("\n".join(output_lines).encode("utf-8")) as file_buffer:
-            file_buffer.filename = f"{batch_id}_{file_type}.jsonl"
-            uploaded_file = await self.files_api.openai_upload_file(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
-            return uploaded_file.id
diff --git a/llama_stack/providers/inline/batches/reference/config.py b/llama_stack/providers/inline/batches/reference/config.py
deleted file mode 100644
index d8d06868b..000000000
--- a/llama_stack/providers/inline/batches/reference/config.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from pydantic import BaseModel, Field
-
-from llama_stack.providers.utils.kvstore.config import KVStoreConfig, SqliteKVStoreConfig
-
-
-class ReferenceBatchesImplConfig(BaseModel):
-    """Configuration for the Reference Batches implementation."""
-
-    kvstore: KVStoreConfig = Field(
-        description="Configuration for the key-value store backend.",
-    )
-
-    max_concurrent_batches: int = Field(
-        default=1,
-        description="Maximum number of concurrent batches to process simultaneously.",
-        ge=1,
-    )
-
-    max_concurrent_requests_per_batch: int = Field(
-        default=10,
-        description="Maximum number of concurrent requests to process per batch.",
-        ge=1,
-    )
-
-    # TODO: add a max requests per second rate limiter
-
-    @classmethod
-    def sample_run_config(cls, __distro_dir__: str) -> dict:
-        return {
-            "kvstore": SqliteKVStoreConfig.sample_run_config(
-                __distro_dir__=__distro_dir__,
-                db_name="batches.db",
-            ),
-        }
diff --git a/llama_stack/providers/registry/batches.py b/llama_stack/providers/registry/batches.py
deleted file mode 100644
index de7886efb..000000000
--- a/llama_stack/providers/registry/batches.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-
-from llama_stack.providers.datatypes import Api, InlineProviderSpec, ProviderSpec
-
-
-def available_providers() -> list[ProviderSpec]:
-    return [
-        InlineProviderSpec(
-            api=Api.batches,
-            provider_type="inline::reference",
-            pip_packages=["openai"],
-            module="llama_stack.providers.inline.batches.reference",
-            config_class="llama_stack.providers.inline.batches.reference.config.ReferenceBatchesImplConfig",
-            api_dependencies=[
-                Api.inference,
-                Api.files,
-                Api.models,
-            ],
-            description="Reference implementation of batches API with KVStore persistence.",
-        ),
-    ]
diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py
index 060acfa72..717677c52 100755
--- a/scripts/provider_codegen.py
+++ b/scripts/provider_codegen.py
@@ -18,23 +18,6 @@ from llama_stack.core.distribution import get_provider_registry
 REPO_ROOT = Path(__file__).parent.parent
 
 
-def get_api_docstring(api_name: str) -> str | None:
-    """Extract docstring from the API protocol class."""
-    try:
-        # Import the API module dynamically
-        api_module = __import__(f"llama_stack.apis.{api_name}", fromlist=[api_name.title()])
-
-        # Get the main protocol class (usually capitalized API name)
-        protocol_class_name = api_name.title()
-        if hasattr(api_module, protocol_class_name):
-            protocol_class = getattr(api_module, protocol_class_name)
-            return protocol_class.__doc__
-    except (ImportError, AttributeError):
-        pass
-
-    return None
-
-
 class ChangedPathTracker:
     """Track a list of paths we may have changed."""
 
@@ -278,11 +261,6 @@ def process_provider_registry(progress, change_tracker: ChangedPathTracker) -> N
             index_content.append(f"# {api_name.title()}\n")
             index_content.append("## Overview\n")
 
-            api_docstring = get_api_docstring(api_name)
-            if api_docstring:
-                cleaned_docstring = api_docstring.strip()
-                index_content.append(f"{cleaned_docstring}\n")
-
             index_content.append(
                 f"This section contains documentation for all available providers for the **{api_name}** API.\n"
             )
diff --git a/tests/integration/batches/__init__.py b/tests/integration/batches/__init__.py
deleted file mode 100644
index 756f351d8..000000000
--- a/tests/integration/batches/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
diff --git a/tests/integration/batches/conftest.py b/tests/integration/batches/conftest.py
deleted file mode 100644
index 974fe77ab..000000000
--- a/tests/integration/batches/conftest.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-"""Shared pytest fixtures for batch tests."""
-
-import json
-import time
-import warnings
-from contextlib import contextmanager
-from io import BytesIO
-
-import pytest
-
-from llama_stack.apis.files import OpenAIFilePurpose
-
-
-class BatchHelper:
-    """Helper class for creating and managing batch input files."""
-
-    def __init__(self, client):
-        """Initialize with either a batch_client or openai_client."""
-        self.client = client
-
-    @contextmanager
-    def create_file(self, content: str | list[dict], filename_prefix="batch_input"):
-        """Context manager for creating and cleaning up batch input files.
-
-        Args:
-            content: Either a list of batch request dictionaries or raw string content
-            filename_prefix: Prefix for the generated filename (or full filename if content is string)
-
-        Yields:
-            The uploaded file object
-        """
-        if isinstance(content, str):
-            # Handle raw string content (e.g., malformed JSONL, empty files)
-            file_content = content.encode("utf-8")
-        else:
-            # Handle list of batch request dictionaries
-            jsonl_content = "\n".join(json.dumps(req) for req in content)
-            file_content = jsonl_content.encode("utf-8")
-
-        filename = filename_prefix if filename_prefix.endswith(".jsonl") else f"{filename_prefix}.jsonl"
-
-        with BytesIO(file_content) as file_buffer:
-            file_buffer.name = filename
-            uploaded_file = self.client.files.create(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
-
-        try:
-            yield uploaded_file
-        finally:
-            try:
-                self.client.files.delete(uploaded_file.id)
-            except Exception:
-                warnings.warn(
-                    f"Failed to cleanup file {uploaded_file.id}: {uploaded_file.filename}",
-                    stacklevel=2,
-                )
-
-    def wait_for(
-        self,
-        batch_id: str,
-        max_wait_time: int = 60,
-        sleep_interval: int | None = None,
-        expected_statuses: set[str] | None = None,
-        timeout_action: str = "fail",
-    ):
-        """Wait for a batch to reach a terminal status.
-
-        Args:
-            batch_id: The batch ID to monitor
-            max_wait_time: Maximum time to wait in seconds (default: 60 seconds)
-            sleep_interval: Time to sleep between checks in seconds (default: 1/10th of max_wait_time, min 1s, max 15s)
-            expected_statuses: Set of expected terminal statuses (default: {"completed"})
-            timeout_action: Action on timeout - "fail" (pytest.fail) or "skip" (pytest.skip)
-
-        Returns:
-            The final batch object
-
-        Raises:
-            pytest.Failed: If batch reaches an unexpected status or timeout_action is "fail"
-            pytest.Skipped: If timeout_action is "skip" on timeout or unexpected status
-        """
-        if sleep_interval is None:
-            # Default to 1/10th of max_wait_time, with min 1s and max 15s
-            sleep_interval = max(1, min(15, max_wait_time // 10))
-
-        if expected_statuses is None:
-            expected_statuses = {"completed"}
-
-        terminal_statuses = {"completed", "failed", "cancelled", "expired"}
-        unexpected_statuses = terminal_statuses - expected_statuses
-
-        start_time = time.time()
-        while time.time() - start_time < max_wait_time:
-            current_batch = self.client.batches.retrieve(batch_id)
-
-            if current_batch.status in expected_statuses:
-                return current_batch
-            elif current_batch.status in unexpected_statuses:
-                error_msg = f"Batch reached unexpected status: {current_batch.status}"
-                if timeout_action == "skip":
-                    pytest.skip(error_msg)
-                else:
-                    pytest.fail(error_msg)
-
-            time.sleep(sleep_interval)
-
-        timeout_msg = f"Batch did not reach expected status {expected_statuses} within {max_wait_time} seconds"
-        if timeout_action == "skip":
-            pytest.skip(timeout_msg)
-        else:
-            pytest.fail(timeout_msg)
-
-
-@pytest.fixture
-def batch_helper(openai_client):
-    """Fixture that provides a BatchHelper instance for OpenAI client."""
-    return BatchHelper(openai_client)
diff --git a/tests/integration/batches/test_batches.py b/tests/integration/batches/test_batches.py
deleted file mode 100644
index 1ef3202d0..000000000
--- a/tests/integration/batches/test_batches.py
+++ /dev/null
@@ -1,270 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-"""
-Integration tests for the Llama Stack batch processing functionality.
-
-This module contains comprehensive integration tests for the batch processing API,
-using the OpenAI-compatible client interface for consistency.
-
-Test Categories:
-    1. Core Batch Operations:
-        - test_batch_creation_and_retrieval: Comprehensive batch creation, structure validation, and retrieval
-        - test_batch_listing: Basic batch listing functionality
-        - test_batch_immediate_cancellation: Batch cancellation workflow
-        # TODO: cancel during processing
-
-    2. End-to-End Processing:
-        - test_batch_e2e_chat_completions: Full chat completions workflow with output and error validation
-
-Note: Error conditions and edge cases are primarily tested in test_batches_errors.py
-for better organization and separation of concerns.
-
-CLEANUP WARNING: These tests currently create batches that are not automatically
-cleaned up after test completion. This may lead to resource accumulation over
-multiple test runs. Only test_batch_immediate_cancellation properly cancels its batch.
-The test_batch_e2e_chat_completions test does clean up its output and error files.
-"""
-
-import json
-
-
-class TestBatchesIntegration:
-    """Integration tests for the batches API."""
-
-    def test_batch_creation_and_retrieval(self, openai_client, batch_helper, text_model_id):
-        """Test comprehensive batch creation and retrieval scenarios."""
-        test_metadata = {
-            "test_type": "comprehensive",
-            "purpose": "creation_and_retrieval_test",
-            "version": "1.0",
-            "tags": "test,batch",
-        }
-
-        batch_requests = [
-            {
-                "custom_id": "request-1",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests, "batch_creation_test") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-                metadata=test_metadata,
-            )
-
-            assert batch.endpoint == "/v1/chat/completions"
-            assert batch.input_file_id == uploaded_file.id
-            assert batch.completion_window == "24h"
-            assert batch.metadata == test_metadata
-
-            retrieved_batch = openai_client.batches.retrieve(batch.id)
-
-            assert retrieved_batch.id == batch.id
-            assert retrieved_batch.object == batch.object
-            assert retrieved_batch.endpoint == batch.endpoint
-            assert retrieved_batch.input_file_id == batch.input_file_id
-            assert retrieved_batch.completion_window == batch.completion_window
-            assert retrieved_batch.metadata == batch.metadata
-
-    def test_batch_listing(self, openai_client, batch_helper, text_model_id):
-        """
-        Test batch listing.
-
-        This test creates multiple batches and verifies that they can be listed.
-        It also deletes the input files before execution, which means the batches
-        will appear as failed due to missing input files. This is expected and
-        a good thing, because it means no inference is performed.
-        """
-        batch_ids = []
-
-        for i in range(2):
-            batch_requests = [
-                {
-                    "custom_id": f"request-{i}",
-                    "method": "POST",
-                    "url": "/v1/chat/completions",
-                    "body": {
-                        "model": text_model_id,
-                        "messages": [{"role": "user", "content": f"Hello {i}"}],
-                        "max_tokens": 10,
-                    },
-                }
-            ]
-
-            with batch_helper.create_file(batch_requests, f"batch_input_{i}") as uploaded_file:
-                batch = openai_client.batches.create(
-                    input_file_id=uploaded_file.id,
-                    endpoint="/v1/chat/completions",
-                    completion_window="24h",
-                )
-                batch_ids.append(batch.id)
-
-        batch_list = openai_client.batches.list()
-
-        assert isinstance(batch_list.data, list)
-
-        listed_batch_ids = {b.id for b in batch_list.data}
-        for batch_id in batch_ids:
-            assert batch_id in listed_batch_ids
-
-    def test_batch_immediate_cancellation(self, openai_client, batch_helper, text_model_id):
-        """Test immediate batch cancellation."""
-        batch_requests = [
-            {
-                "custom_id": "request-1",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests) as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            # hopefully cancel the batch before it completes
-            cancelling_batch = openai_client.batches.cancel(batch.id)
-            assert cancelling_batch.status in ["cancelling", "cancelled"]
-            assert isinstance(cancelling_batch.cancelling_at, int), (
-                f"cancelling_at should be int, got {type(cancelling_batch.cancelling_at)}"
-            )
-
-            final_batch = batch_helper.wait_for(
-                batch.id,
-                max_wait_time=3 * 60,  # often takes 10-11 minutes, give it 3 min
-                expected_statuses={"cancelled"},
-                timeout_action="skip",
-            )
-
-        assert final_batch.status == "cancelled"
-        assert isinstance(final_batch.cancelled_at, int), (
-            f"cancelled_at should be int, got {type(final_batch.cancelled_at)}"
-        )
-
-    def test_batch_e2e_chat_completions(self, openai_client, batch_helper, text_model_id):
-        """Test end-to-end batch processing for chat completions with both successful and failed operations."""
-        batch_requests = [
-            {
-                "custom_id": "success-1",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Say hello"}],
-                    "max_tokens": 20,
-                },
-            },
-            {
-                "custom_id": "error-1",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "This should fail"}],
-                    "max_tokens": -1,  # Invalid negative max_tokens will cause inference error
-                },
-            },
-        ]
-
-        with batch_helper.create_file(batch_requests) as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-                metadata={"test": "e2e_success_and_errors_test"},
-            )
-
-            final_batch = batch_helper.wait_for(
-                batch.id,
-                max_wait_time=3 * 60,  # often takes 2-3 minutes
-                expected_statuses={"completed"},
-                timeout_action="skip",
-            )
-
-        # Expecting a completed batch with both successful and failed requests
-        #  Batch(id='batch_xxx',
-        #        completion_window='24h',
-        #        created_at=...,
-        #        endpoint='/v1/chat/completions',
-        #        input_file_id='file-xxx',
-        #        object='batch',
-        #        status='completed',
-        #        output_file_id='file-xxx',
-        #        error_file_id='file-xxx',
-        #        request_counts=BatchRequestCounts(completed=1, failed=1, total=2))
-
-        assert final_batch.status == "completed"
-        assert final_batch.request_counts is not None
-        assert final_batch.request_counts.total == 2
-        assert final_batch.request_counts.completed == 1
-        assert final_batch.request_counts.failed == 1
-
-        assert final_batch.output_file_id is not None, "Output file should exist for successful requests"
-
-        output_content = openai_client.files.content(final_batch.output_file_id)
-        if isinstance(output_content, str):
-            output_text = output_content
-        else:
-            output_text = output_content.content.decode("utf-8")
-
-        output_lines = output_text.strip().split("\n")
-
-        for line in output_lines:
-            result = json.loads(line)
-
-            assert "id" in result
-            assert "custom_id" in result
-            assert result["custom_id"] == "success-1"
-
-            assert "response" in result
-
-            assert result["response"]["status_code"] == 200
-            assert "body" in result["response"]
-            assert "choices" in result["response"]["body"]
-
-        assert final_batch.error_file_id is not None, "Error file should exist for failed requests"
-
-        error_content = openai_client.files.content(final_batch.error_file_id)
-        if isinstance(error_content, str):
-            error_text = error_content
-        else:
-            error_text = error_content.content.decode("utf-8")
-
-        error_lines = error_text.strip().split("\n")
-
-        for line in error_lines:
-            result = json.loads(line)
-
-            assert "id" in result
-            assert "custom_id" in result
-            assert result["custom_id"] == "error-1"
-            assert "error" in result
-            error = result["error"]
-            assert error is not None
-            assert "code" in error or "message" in error, "Error should have code or message"
-
-        deleted_output_file = openai_client.files.delete(final_batch.output_file_id)
-        assert deleted_output_file.deleted, f"Output file {final_batch.output_file_id} was not deleted successfully"
-
-        deleted_error_file = openai_client.files.delete(final_batch.error_file_id)
-        assert deleted_error_file.deleted, f"Error file {final_batch.error_file_id} was not deleted successfully"
diff --git a/tests/integration/batches/test_batches_errors.py b/tests/integration/batches/test_batches_errors.py
deleted file mode 100644
index bc94a182e..000000000
--- a/tests/integration/batches/test_batches_errors.py
+++ /dev/null
@@ -1,693 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-"""
-Error handling and edge case tests for the Llama Stack batch processing functionality.
-
-This module focuses exclusively on testing error conditions, validation failures,
-and edge cases for batch operations to ensure robust error handling and graceful
-degradation.
-
-Test Categories:
-    1. File and Input Validation:
-        - test_batch_nonexistent_file_id: Handling invalid file IDs
-        - test_batch_malformed_jsonl: Processing malformed JSONL input files
-        - test_file_malformed_batch_file: Handling malformed files at upload time
-        - test_batch_missing_required_fields: Validation of required request fields
-
-    2. API Endpoint and Model Validation:
-        - test_batch_invalid_endpoint: Invalid endpoint handling during creation
-        - test_batch_error_handling_invalid_model: Error handling with nonexistent models
-        - test_batch_endpoint_mismatch: Validation of endpoint/URL consistency
-
-    3. Batch Lifecycle Error Handling:
-        - test_batch_retrieve_nonexistent: Retrieving non-existent batches
-        - test_batch_cancel_nonexistent: Cancelling non-existent batches
-        - test_batch_cancel_completed: Attempting to cancel completed batches
-
-    4. Parameter and Configuration Validation:
-        - test_batch_invalid_completion_window: Invalid completion window values
-        - test_batch_invalid_metadata_types: Invalid metadata type validation
-        - test_batch_missing_required_body_fields: Validation of required fields in request body
-
-    5. Feature Restriction and Compatibility:
-        - test_batch_streaming_not_supported: Streaming request rejection
-        - test_batch_mixed_streaming_requests: Mixed streaming/non-streaming validation
-
-Note: Core functionality and OpenAI compatibility tests are located in
-test_batches_integration.py for better organization and separation of concerns.
-
-CLEANUP WARNING: These tests create batches to test error conditions but do not
-automatically clean them up after test completion. While most error tests create
-batches that fail quickly, some may create valid batches that consume resources.
-"""
-
-import pytest
-from openai import BadRequestError, ConflictError, NotFoundError
-
-
-class TestBatchesErrorHandling:
-    """Error handling and edge case tests for the batches API using OpenAI client."""
-
-    def test_batch_nonexistent_file_id(self, openai_client, batch_helper):
-        """Test batch creation with nonexistent input file ID."""
-
-        batch = openai_client.batches.create(
-            input_file_id="file-nonexistent-xyz",
-            endpoint="/v1/chat/completions",
-            completion_window="24h",
-        )
-
-        final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #      BatchError(
-        #        code='invalid_request',
-        #        line=None,
-        #        message='Cannot find file ..., or organization ... does not have access to it.',
-        #        param='file_id')
-        #    ], object='list'),
-        #    failed_at=1754566971,
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 1
-        error = final_batch.errors.data[0]
-        assert error.code == "invalid_request"
-        assert "cannot find file" in error.message.lower()
-
-    def test_batch_invalid_endpoint(self, openai_client, batch_helper, text_model_id):
-        """Test batch creation with invalid endpoint."""
-        batch_requests = [
-            {
-                "custom_id": "invalid-endpoint",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests) as uploaded_file:
-            with pytest.raises(BadRequestError) as exc_info:
-                openai_client.batches.create(
-                    input_file_id=uploaded_file.id,
-                    endpoint="/v1/invalid/endpoint",
-                    completion_window="24h",
-                )
-
-            # Expected -
-            #  Error code: 400 - {
-            #    'error': {
-            #      'message': "Invalid value: '/v1/invalid/endpoint'. Supported values are: '/v1/chat/completions', '/v1/completions', '/v1/embeddings', and '/v1/responses'.",
-            #      'type': 'invalid_request_error',
-            #      'param': 'endpoint',
-            #      'code': 'invalid_value'
-            #    }
-            #  }
-
-            error_msg = str(exc_info.value).lower()
-            assert exc_info.value.status_code == 400
-            assert "invalid value" in error_msg
-            assert "/v1/invalid/endpoint" in error_msg
-            assert "supported values" in error_msg
-            assert "endpoint" in error_msg
-            assert "invalid_value" in error_msg
-
-    def test_batch_malformed_jsonl(self, openai_client, batch_helper):
-        """
-        Test batch with malformed JSONL input.
-
-        The /v1/files endpoint requires valid JSONL format, so we provide a well formed line
-        before a malformed line to ensure we get to the /v1/batches validation stage.
-        """
-        with batch_helper.create_file(
-            """{"custom_id": "valid", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "test"}}
-{invalid json here""",
-            "malformed_batch_input.jsonl",
-        ) as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #      ...,
-        #      BatchError(code='invalid_json_line',
-        #                 line=2,
-        #                 message='This line is not parseable as valid JSON.',
-        #                 param=None)
-        #    ], object='list'),
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) > 0
-        error = final_batch.errors.data[-1]  # get last error because first may be about the "test" model
-        assert error.code == "invalid_json_line"
-        assert error.line == 2
-        assert "not" in error.message.lower()
-        assert "valid json" in error.message.lower()
-
-    @pytest.mark.xfail(reason="Not all file providers validate content")
-    @pytest.mark.parametrize("batch_requests", ["", "{malformed json"], ids=["empty", "malformed"])
-    def test_file_malformed_batch_file(self, openai_client, batch_helper, batch_requests):
-        """Test file upload with malformed content."""
-
-        with pytest.raises(BadRequestError) as exc_info:
-            with batch_helper.create_file(batch_requests, "malformed_batch_input_file.jsonl"):
-                # /v1/files rejects the file, we don't get to batch creation
-                pass
-
-        error_msg = str(exc_info.value).lower()
-        assert exc_info.value.status_code == 400
-        assert "invalid file format" in error_msg
-        assert "jsonl" in error_msg
-
-    def test_batch_retrieve_nonexistent(self, openai_client):
-        """Test retrieving nonexistent batch."""
-        with pytest.raises(NotFoundError) as exc_info:
-            openai_client.batches.retrieve("batch-nonexistent-xyz")
-
-        error_msg = str(exc_info.value).lower()
-        assert exc_info.value.status_code == 404
-        assert "no batch found" in error_msg or "not found" in error_msg
-
-    def test_batch_cancel_nonexistent(self, openai_client):
-        """Test cancelling nonexistent batch."""
-        with pytest.raises(NotFoundError) as exc_info:
-            openai_client.batches.cancel("batch-nonexistent-xyz")
-
-        error_msg = str(exc_info.value).lower()
-        assert exc_info.value.status_code == 404
-        assert "no batch found" in error_msg or "not found" in error_msg
-
-    def test_batch_cancel_completed(self, openai_client, batch_helper, text_model_id):
-        """Test cancelling already completed batch."""
-        batch_requests = [
-            {
-                "custom_id": "cancel-completed",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Quick test"}],
-                    "max_tokens": 5,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests, "cancel_test_batch_input") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(
-                batch.id,
-                max_wait_time=3 * 60,  # often take 10-11 min, give it 3 min
-                expected_statuses={"completed"},
-                timeout_action="skip",
-            )
-
-        deleted_file = openai_client.files.delete(final_batch.output_file_id)
-        assert deleted_file.deleted, f"File {final_batch.output_file_id} was not deleted successfully"
-
-        with pytest.raises(ConflictError) as exc_info:
-            openai_client.batches.cancel(batch.id)
-
-        # Expecting -
-        #   Error code: 409 - {
-        #     'error': {
-        #       'message': "Cannot cancel a batch with status 'completed'.",
-        #       'type': 'invalid_request_error',
-        #       'param': None,
-        #       'code': None
-        #     }
-        #   }
-        #
-        # NOTE: Same for "failed", cancelling "cancelled" batches is allowed
-
-        error_msg = str(exc_info.value).lower()
-        assert exc_info.value.status_code == 409
-        assert "cannot cancel" in error_msg
-
-    def test_batch_missing_required_fields(self, openai_client, batch_helper, text_model_id):
-        """Test batch with requests missing required fields."""
-        batch_requests = [
-            {
-                # Missing custom_id
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "No custom_id"}],
-                    "max_tokens": 10,
-                },
-            },
-            {
-                "custom_id": "no-method",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "No method"}],
-                    "max_tokens": 10,
-                },
-            },
-            {
-                "custom_id": "no-url",
-                "method": "POST",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "No URL"}],
-                    "max_tokens": 10,
-                },
-            },
-            {
-                "custom_id": "no-body",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-            },
-        ]
-
-        with batch_helper.create_file(batch_requests, "missing_fields_batch_input") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(
-        #      data=[
-        #        BatchError(
-        #          code='missing_required_parameter',
-        #          line=1,
-        #          message="Missing required parameter: 'custom_id'.",
-        #          param='custom_id'
-        #        ),
-        #        BatchError(
-        #          code='missing_required_parameter',
-        #          line=2,
-        #          message="Missing required parameter: 'method'.",
-        #          param='method'
-        #        ),
-        #        BatchError(
-        #          code='missing_required_parameter',
-        #          line=3,
-        #          message="Missing required parameter: 'url'.",
-        #          param='url'
-        #        ),
-        #        BatchError(
-        #          code='missing_required_parameter',
-        #          line=4,
-        #          message="Missing required parameter: 'body'.",
-        #          param='body'
-        #        )
-        #    ], object='list'),
-        #    failed_at=1754566945,
-        #    ...)
-        #  )
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 4
-        no_custom_id_error = final_batch.errors.data[0]
-        assert no_custom_id_error.code == "missing_required_parameter"
-        assert no_custom_id_error.line == 1
-        assert "missing" in no_custom_id_error.message.lower()
-        assert "custom_id" in no_custom_id_error.message.lower()
-        no_method_error = final_batch.errors.data[1]
-        assert no_method_error.code == "missing_required_parameter"
-        assert no_method_error.line == 2
-        assert "missing" in no_method_error.message.lower()
-        assert "method" in no_method_error.message.lower()
-        no_url_error = final_batch.errors.data[2]
-        assert no_url_error.code == "missing_required_parameter"
-        assert no_url_error.line == 3
-        assert "missing" in no_url_error.message.lower()
-        assert "url" in no_url_error.message.lower()
-        no_body_error = final_batch.errors.data[3]
-        assert no_body_error.code == "missing_required_parameter"
-        assert no_body_error.line == 4
-        assert "missing" in no_body_error.message.lower()
-        assert "body" in no_body_error.message.lower()
-
-    def test_batch_invalid_completion_window(self, openai_client, batch_helper, text_model_id):
-        """Test batch creation with invalid completion window."""
-        batch_requests = [
-            {
-                "custom_id": "invalid-completion-window",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests) as uploaded_file:
-            for window in ["1h", "48h", "invalid", ""]:
-                with pytest.raises(BadRequestError) as exc_info:
-                    openai_client.batches.create(
-                        input_file_id=uploaded_file.id,
-                        endpoint="/v1/chat/completions",
-                        completion_window=window,
-                    )
-            assert exc_info.value.status_code == 400
-            error_msg = str(exc_info.value).lower()
-            assert "error" in error_msg
-            assert "completion_window" in error_msg
-
-    def test_batch_streaming_not_supported(self, openai_client, batch_helper, text_model_id):
-        """Test that streaming responses are not supported in batches."""
-        batch_requests = [
-            {
-                "custom_id": "streaming-test",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                    "stream": True,  # Not supported
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests, "streaming_batch_input") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #       BatchError(code='streaming_unsupported',
-        #         line=1,
-        #         message='Chat Completions: Streaming is not supported in the Batch API.',
-        #         param='body.stream')
-        #    ], object='list'),
-        #    failed_at=1754566965,
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 1
-        error = final_batch.errors.data[0]
-        assert error.code == "streaming_unsupported"
-        assert error.line == 1
-        assert "streaming" in error.message.lower()
-        assert "not supported" in error.message.lower()
-        assert error.param == "body.stream"
-        assert final_batch.failed_at is not None
-
-    def test_batch_mixed_streaming_requests(self, openai_client, batch_helper, text_model_id):
-        """
-        Test batch with mixed streaming and non-streaming requests.
-
-        This is distinct from test_batch_streaming_not_supported, which tests a single
-        streaming request, to ensure an otherwise valid batch fails when a single
-        streaming request is included.
-        """
-        batch_requests = [
-            {
-                "custom_id": "valid-non-streaming-request",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello without streaming"}],
-                    "max_tokens": 10,
-                },
-            },
-            {
-                "custom_id": "streaming-request",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello with streaming"}],
-                    "max_tokens": 10,
-                    "stream": True,  # Not supported
-                },
-            },
-        ]
-
-        with batch_helper.create_file(batch_requests, "mixed_streaming_batch_input") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #      BatchError(
-        #        code='streaming_unsupported',
-        #        line=2,
-        #        message='Chat Completions: Streaming is not supported in the Batch API.',
-        #        param='body.stream')
-        #    ], object='list'),
-        #    failed_at=1754574442,
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 1
-        error = final_batch.errors.data[0]
-        assert error.code == "streaming_unsupported"
-        assert error.line == 2
-        assert "streaming" in error.message.lower()
-        assert "not supported" in error.message.lower()
-        assert error.param == "body.stream"
-        assert final_batch.failed_at is not None
-
-    def test_batch_endpoint_mismatch(self, openai_client, batch_helper, text_model_id):
-        """Test batch creation with mismatched endpoint and request URL."""
-        batch_requests = [
-            {
-                "custom_id": "endpoint-mismatch",
-                "method": "POST",
-                "url": "/v1/embeddings",  # Different from batch endpoint
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests, "endpoint_mismatch_batch_input") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",  # Different from request URL
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #      BatchError(
-        #        code='invalid_url',
-        #        line=1,
-        #        message='The URL provided for this request does not match the batch endpoint.',
-        #        param='url')
-        #    ], object='list'),
-        #    failed_at=1754566972,
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 1
-        error = final_batch.errors.data[0]
-        assert error.line == 1
-        assert error.code == "invalid_url"
-        assert "does not match" in error.message.lower()
-        assert "endpoint" in error.message.lower()
-        assert final_batch.failed_at is not None
-
-    def test_batch_error_handling_invalid_model(self, openai_client, batch_helper):
-        """Test batch error handling with invalid model."""
-        batch_requests = [
-            {
-                "custom_id": "invalid-model",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": "nonexistent-model-xyz",
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests) as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #      BatchError(code='model_not_found',
-        #        line=1,
-        #        message="The provided model 'nonexistent-model-xyz' is not supported by the Batch API.",
-        #        param='body.model')
-        #    ], object='list'),
-        #    failed_at=1754566978,
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 1
-        error = final_batch.errors.data[0]
-        assert error.line == 1
-        assert error.code == "model_not_found"
-        assert "not supported" in error.message.lower()
-        assert error.param == "body.model"
-        assert final_batch.failed_at is not None
-
-    def test_batch_missing_required_body_fields(self, openai_client, batch_helper, text_model_id):
-        """Test batch with requests missing required fields in body (model and messages)."""
-        batch_requests = [
-            {
-                "custom_id": "missing-model",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    # Missing model field
-                    "messages": [{"role": "user", "content": "Hello without model"}],
-                    "max_tokens": 10,
-                },
-            },
-            {
-                "custom_id": "missing-messages",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    # Missing messages field
-                    "max_tokens": 10,
-                },
-            },
-        ]
-
-        with batch_helper.create_file(batch_requests, "missing_body_fields_batch_input") as uploaded_file:
-            batch = openai_client.batches.create(
-                input_file_id=uploaded_file.id,
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-            )
-
-            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
-
-        # Expecting -
-        #  Batch(...,
-        #    status='failed',
-        #    errors=Errors(data=[
-        #      BatchError(
-        #        code='invalid_request',
-        #        line=1,
-        #        message='Model parameter is required.',
-        #        param='body.model'),
-        #      BatchError(
-        #        code='invalid_request',
-        #        line=2,
-        #        message='Messages parameter is required.',
-        #        param='body.messages')
-        #      ], object='list'),
-        #    ...)
-
-        assert final_batch.status == "failed"
-        assert final_batch.errors is not None
-        assert len(final_batch.errors.data) == 2
-
-        model_error = final_batch.errors.data[0]
-        assert model_error.line == 1
-        assert "model" in model_error.message.lower()
-        assert model_error.param == "body.model"
-
-        messages_error = final_batch.errors.data[1]
-        assert messages_error.line == 2
-        assert "messages" in messages_error.message.lower()
-        assert messages_error.param == "body.messages"
-
-        assert final_batch.failed_at is not None
-
-    def test_batch_invalid_metadata_types(self, openai_client, batch_helper, text_model_id):
-        """Test batch creation with invalid metadata types (like lists)."""
-        batch_requests = [
-            {
-                "custom_id": "invalid-metadata-type",
-                "method": "POST",
-                "url": "/v1/chat/completions",
-                "body": {
-                    "model": text_model_id,
-                    "messages": [{"role": "user", "content": "Hello"}],
-                    "max_tokens": 10,
-                },
-            }
-        ]
-
-        with batch_helper.create_file(batch_requests) as uploaded_file:
-            with pytest.raises(Exception) as exc_info:
-                openai_client.batches.create(
-                    input_file_id=uploaded_file.id,
-                    endpoint="/v1/chat/completions",
-                    completion_window="24h",
-                    metadata={
-                        "tags": ["tag1", "tag2"],  # Invalid type, should be a string
-                    },
-                )
-
-        # Expecting -
-        #  Error code: 400 - {'error':
-        #    {'message': "Invalid type for 'metadata.tags': expected a string,
-        #                 but got an array instead.",
-        #     'type': 'invalid_request_error', 'param': 'metadata.tags',
-        #     'code': 'invalid_type'}}
-
-        error_msg = str(exc_info.value).lower()
-        assert "400" in error_msg
-        assert "tags" in error_msg
-        assert "string" in error_msg
diff --git a/tests/unit/providers/batches/test_reference.py b/tests/unit/providers/batches/test_reference.py
deleted file mode 100644
index 9fe0cc710..000000000
--- a/tests/unit/providers/batches/test_reference.py
+++ /dev/null
@@ -1,753 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-"""
-Test suite for the reference implementation of the Batches API.
-
-The tests are categorized and outlined below, keep this updated:
-
-- Batch creation with various parameters and validation:
-  * test_create_and_retrieve_batch_success (positive)
-  * test_create_batch_without_metadata (positive)
-  * test_create_batch_completion_window (negative)
-  * test_create_batch_invalid_endpoints (negative)
-  * test_create_batch_invalid_metadata (negative)
-
-- Batch retrieval and error handling for non-existent batches:
-  * test_retrieve_batch_not_found (negative)
-
-- Batch cancellation with proper status transitions:
-  * test_cancel_batch_success (positive)
-  * test_cancel_batch_invalid_statuses (negative)
-  * test_cancel_batch_not_found (negative)
-
-- Batch listing with pagination and filtering:
-  * test_list_batches_empty (positive)
-  * test_list_batches_single_batch (positive)
-  * test_list_batches_multiple_batches (positive)
-  * test_list_batches_with_limit (positive)
-  * test_list_batches_with_pagination (positive)
-  * test_list_batches_invalid_after (negative)
-
-- Data persistence in the underlying key-value store:
-  * test_kvstore_persistence (positive)
-
-- Batch processing concurrency control:
-  * test_max_concurrent_batches (positive)
-
-- Input validation testing (direct _validate_input method tests):
-  * test_validate_input_file_not_found (negative)
-  * test_validate_input_file_exists_empty_content (positive)
-  * test_validate_input_file_mixed_valid_invalid_json (mixed)
-  * test_validate_input_invalid_model (negative)
-  * test_validate_input_url_mismatch (negative)
-  * test_validate_input_multiple_errors_per_request (negative)
-  * test_validate_input_invalid_request_format (negative)
-  * test_validate_input_missing_parameters (parametrized negative - custom_id, method, url, body, model, messages missing validation)
-  * test_validate_input_invalid_parameter_types (parametrized negative - custom_id, url, method, body, model, messages type validation)
-
-The tests use temporary SQLite databases for isolation and mock external
-dependencies like inference, files, and models APIs.
-"""
-
-import json
-import tempfile
-from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from llama_stack.apis.batches import BatchObject
-from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
-from llama_stack.providers.inline.batches.reference.batches import ReferenceBatchesImpl
-from llama_stack.providers.inline.batches.reference.config import ReferenceBatchesImplConfig
-from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig
-
-
-class TestReferenceBatchesImpl:
-    """Test the reference implementation of the Batches API."""
-
-    @pytest.fixture
-    async def provider(self):
-        """Create a test provider instance with temporary database."""
-        with tempfile.TemporaryDirectory() as tmpdir:
-            db_path = Path(tmpdir) / "test_batches.db"
-            kvstore_config = SqliteKVStoreConfig(db_path=str(db_path))
-            config = ReferenceBatchesImplConfig(kvstore=kvstore_config)
-
-            # Create kvstore and mock APIs
-            from unittest.mock import AsyncMock
-
-            from llama_stack.providers.utils.kvstore import kvstore_impl
-
-            kvstore = await kvstore_impl(config.kvstore)
-            mock_inference = AsyncMock()
-            mock_files = AsyncMock()
-            mock_models = AsyncMock()
-
-            provider = ReferenceBatchesImpl(config, mock_inference, mock_files, mock_models, kvstore)
-            await provider.initialize()
-
-            # unit tests should not require background processing
-            provider.process_batches = False
-
-            yield provider
-
-            await provider.shutdown()
-
-    @pytest.fixture
-    def sample_batch_data(self):
-        """Sample batch data for testing."""
-        return {
-            "input_file_id": "file_abc123",
-            "endpoint": "/v1/chat/completions",
-            "completion_window": "24h",
-            "metadata": {"test": "true", "priority": "high"},
-        }
-
-    def _validate_batch_type(self, batch, expected_metadata=None):
-        """
-        Helper function to validate batch object structure and field types.
-
-        Note: This validates the direct BatchObject from the provider, not the
-              client library response which has a different structure.
-
-        Args:
-            batch: The BatchObject instance to validate.
-            expected_metadata: Optional expected metadata dictionary to validate against.
-        """
-        assert isinstance(batch.id, str)
-        assert isinstance(batch.completion_window, str)
-        assert isinstance(batch.created_at, int)
-        assert isinstance(batch.endpoint, str)
-        assert isinstance(batch.input_file_id, str)
-        assert batch.object == "batch"
-        assert batch.status in [
-            "validating",
-            "failed",
-            "in_progress",
-            "finalizing",
-            "completed",
-            "expired",
-            "cancelling",
-            "cancelled",
-        ]
-
-        if expected_metadata is not None:
-            assert batch.metadata == expected_metadata
-
-        timestamp_fields = [
-            "cancelled_at",
-            "cancelling_at",
-            "completed_at",
-            "expired_at",
-            "expires_at",
-            "failed_at",
-            "finalizing_at",
-            "in_progress_at",
-        ]
-        for field in timestamp_fields:
-            field_value = getattr(batch, field, None)
-            if field_value is not None:
-                assert isinstance(field_value, int), f"{field} should be int or None, got {type(field_value)}"
-
-        file_id_fields = ["error_file_id", "output_file_id"]
-        for field in file_id_fields:
-            field_value = getattr(batch, field, None)
-            if field_value is not None:
-                assert isinstance(field_value, str), f"{field} should be str or None, got {type(field_value)}"
-
-        if hasattr(batch, "request_counts") and batch.request_counts is not None:
-            assert isinstance(batch.request_counts.completed, int), (
-                f"request_counts.completed should be int, got {type(batch.request_counts.completed)}"
-            )
-            assert isinstance(batch.request_counts.failed, int), (
-                f"request_counts.failed should be int, got {type(batch.request_counts.failed)}"
-            )
-            assert isinstance(batch.request_counts.total, int), (
-                f"request_counts.total should be int, got {type(batch.request_counts.total)}"
-            )
-
-        if hasattr(batch, "errors") and batch.errors is not None:
-            assert isinstance(batch.errors, dict), f"errors should be object or dict, got {type(batch.errors)}"
-
-            if hasattr(batch.errors, "data") and batch.errors.data is not None:
-                assert isinstance(batch.errors.data, list), (
-                    f"errors.data should be list or None, got {type(batch.errors.data)}"
-                )
-
-                for i, error_item in enumerate(batch.errors.data):
-                    assert isinstance(error_item, dict), (
-                        f"errors.data[{i}] should be object or dict, got {type(error_item)}"
-                    )
-
-                    if hasattr(error_item, "code") and error_item.code is not None:
-                        assert isinstance(error_item.code, str), (
-                            f"errors.data[{i}].code should be str or None, got {type(error_item.code)}"
-                        )
-
-                    if hasattr(error_item, "line") and error_item.line is not None:
-                        assert isinstance(error_item.line, int), (
-                            f"errors.data[{i}].line should be int or None, got {type(error_item.line)}"
-                        )
-
-                    if hasattr(error_item, "message") and error_item.message is not None:
-                        assert isinstance(error_item.message, str), (
-                            f"errors.data[{i}].message should be str or None, got {type(error_item.message)}"
-                        )
-
-                    if hasattr(error_item, "param") and error_item.param is not None:
-                        assert isinstance(error_item.param, str), (
-                            f"errors.data[{i}].param should be str or None, got {type(error_item.param)}"
-                        )
-
-            if hasattr(batch.errors, "object") and batch.errors.object is not None:
-                assert isinstance(batch.errors.object, str), (
-                    f"errors.object should be str or None, got {type(batch.errors.object)}"
-                )
-                assert batch.errors.object == "list", f"errors.object should be 'list', got {batch.errors.object}"
-
-    async def test_create_and_retrieve_batch_success(self, provider, sample_batch_data):
-        """Test successful batch creation and retrieval."""
-        created_batch = await provider.create_batch(**sample_batch_data)
-
-        self._validate_batch_type(created_batch, expected_metadata=sample_batch_data["metadata"])
-
-        assert created_batch.id.startswith("batch_")
-        assert len(created_batch.id) > 13
-        assert created_batch.object == "batch"
-        assert created_batch.endpoint == sample_batch_data["endpoint"]
-        assert created_batch.input_file_id == sample_batch_data["input_file_id"]
-        assert created_batch.completion_window == sample_batch_data["completion_window"]
-        assert created_batch.status == "validating"
-        assert created_batch.metadata == sample_batch_data["metadata"]
-        assert isinstance(created_batch.created_at, int)
-        assert created_batch.created_at > 0
-
-        retrieved_batch = await provider.retrieve_batch(created_batch.id)
-
-        self._validate_batch_type(retrieved_batch, expected_metadata=sample_batch_data["metadata"])
-
-        assert retrieved_batch.id == created_batch.id
-        assert retrieved_batch.input_file_id == created_batch.input_file_id
-        assert retrieved_batch.endpoint == created_batch.endpoint
-        assert retrieved_batch.status == created_batch.status
-        assert retrieved_batch.metadata == created_batch.metadata
-
-    async def test_create_batch_without_metadata(self, provider):
-        """Test batch creation without optional metadata."""
-        batch = await provider.create_batch(
-            input_file_id="file_123", endpoint="/v1/chat/completions", completion_window="24h"
-        )
-
-        assert batch.metadata is None
-
-    async def test_create_batch_completion_window(self, provider):
-        """Test batch creation with invalid completion window."""
-        with pytest.raises(ValueError, match="Invalid completion_window"):
-            await provider.create_batch(
-                input_file_id="file_123", endpoint="/v1/chat/completions", completion_window="now"
-            )
-
-    @pytest.mark.parametrize(
-        "endpoint",
-        [
-            "/v1/embeddings",
-            "/v1/completions",
-            "/v1/invalid/endpoint",
-            "",
-        ],
-    )
-    async def test_create_batch_invalid_endpoints(self, provider, endpoint):
-        """Test batch creation with various invalid endpoints."""
-        with pytest.raises(ValueError, match="Invalid endpoint"):
-            await provider.create_batch(input_file_id="file_123", endpoint=endpoint, completion_window="24h")
-
-    async def test_create_batch_invalid_metadata(self, provider):
-        """Test that batch creation fails with invalid metadata."""
-        with pytest.raises(ValueError, match="should be a valid string"):
-            await provider.create_batch(
-                input_file_id="file_123",
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-                metadata={123: "invalid_key"},  # Non-string key
-            )
-
-        with pytest.raises(ValueError, match="should be a valid string"):
-            await provider.create_batch(
-                input_file_id="file_123",
-                endpoint="/v1/chat/completions",
-                completion_window="24h",
-                metadata={"valid_key": 456},  # Non-string value
-            )
-
-    async def test_retrieve_batch_not_found(self, provider):
-        """Test error when retrieving non-existent batch."""
-        with pytest.raises(ResourceNotFoundError, match=r"Batch 'nonexistent_batch' not found"):
-            await provider.retrieve_batch("nonexistent_batch")
-
-    async def test_cancel_batch_success(self, provider, sample_batch_data):
-        """Test successful batch cancellation."""
-        created_batch = await provider.create_batch(**sample_batch_data)
-        assert created_batch.status == "validating"
-
-        cancelled_batch = await provider.cancel_batch(created_batch.id)
-
-        assert cancelled_batch.id == created_batch.id
-        assert cancelled_batch.status in ["cancelling", "cancelled"]
-        assert isinstance(cancelled_batch.cancelling_at, int)
-        assert cancelled_batch.cancelling_at >= created_batch.created_at
-
-    @pytest.mark.parametrize("status", ["failed", "expired", "completed"])
-    async def test_cancel_batch_invalid_statuses(self, provider, sample_batch_data, status):
-        """Test error when cancelling batch in final states."""
-        provider.process_batches = False
-        created_batch = await provider.create_batch(**sample_batch_data)
-
-        # directly update status in kvstore
-        await provider._update_batch(created_batch.id, status=status)
-
-        with pytest.raises(ConflictError, match=f"Cannot cancel batch '{created_batch.id}' with status '{status}'"):
-            await provider.cancel_batch(created_batch.id)
-
-    async def test_cancel_batch_not_found(self, provider):
-        """Test error when cancelling non-existent batch."""
-        with pytest.raises(ResourceNotFoundError, match=r"Batch 'nonexistent_batch' not found"):
-            await provider.cancel_batch("nonexistent_batch")
-
-    async def test_list_batches_empty(self, provider):
-        """Test listing batches when none exist."""
-        response = await provider.list_batches()
-
-        assert response.object == "list"
-        assert response.data == []
-        assert response.first_id is None
-        assert response.last_id is None
-        assert response.has_more is False
-
-    async def test_list_batches_single_batch(self, provider, sample_batch_data):
-        """Test listing batches with single batch."""
-        created_batch = await provider.create_batch(**sample_batch_data)
-
-        response = await provider.list_batches()
-
-        assert len(response.data) == 1
-        self._validate_batch_type(response.data[0], expected_metadata=sample_batch_data["metadata"])
-        assert response.data[0].id == created_batch.id
-        assert response.first_id == created_batch.id
-        assert response.last_id == created_batch.id
-        assert response.has_more is False
-
-    async def test_list_batches_multiple_batches(self, provider):
-        """Test listing multiple batches."""
-        batches = [
-            await provider.create_batch(
-                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
-            )
-            for i in range(3)
-        ]
-
-        response = await provider.list_batches()
-
-        assert len(response.data) == 3
-
-        batch_ids = {batch.id for batch in response.data}
-        expected_ids = {batch.id for batch in batches}
-        assert batch_ids == expected_ids
-        assert response.has_more is False
-
-        assert response.first_id in expected_ids
-        assert response.last_id in expected_ids
-
-    async def test_list_batches_with_limit(self, provider):
-        """Test listing batches with limit parameter."""
-        batches = [
-            await provider.create_batch(
-                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
-            )
-            for i in range(3)
-        ]
-
-        response = await provider.list_batches(limit=2)
-
-        assert len(response.data) == 2
-        assert response.has_more is True
-        assert response.first_id == response.data[0].id
-        assert response.last_id == response.data[1].id
-        batch_ids = {batch.id for batch in response.data}
-        expected_ids = {batch.id for batch in batches}
-        assert batch_ids.issubset(expected_ids)
-
-    async def test_list_batches_with_pagination(self, provider):
-        """Test listing batches with pagination using 'after' parameter."""
-        for i in range(3):
-            await provider.create_batch(
-                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
-            )
-
-        # Get first page
-        first_page = await provider.list_batches(limit=1)
-        assert len(first_page.data) == 1
-        assert first_page.has_more is True
-
-        # Get second page using 'after'
-        second_page = await provider.list_batches(limit=1, after=first_page.data[0].id)
-        assert len(second_page.data) == 1
-        assert second_page.data[0].id != first_page.data[0].id
-
-        # Verify we got the next batch in order
-        all_batches = await provider.list_batches()
-        expected_second_batch_id = all_batches.data[1].id
-        assert second_page.data[0].id == expected_second_batch_id
-
-    async def test_list_batches_invalid_after(self, provider, sample_batch_data):
-        """Test listing batches with invalid 'after' parameter."""
-        await provider.create_batch(**sample_batch_data)
-
-        response = await provider.list_batches(after="nonexistent_batch")
-
-        # Should return all batches (no filtering when 'after' batch not found)
-        assert len(response.data) == 1
-
-    async def test_kvstore_persistence(self, provider, sample_batch_data):
-        """Test that batches are properly persisted in kvstore."""
-        batch = await provider.create_batch(**sample_batch_data)
-
-        stored_data = await provider.kvstore.get(f"batch:{batch.id}")
-        assert stored_data is not None
-
-        stored_batch_dict = json.loads(stored_data)
-        assert stored_batch_dict["id"] == batch.id
-        assert stored_batch_dict["input_file_id"] == sample_batch_data["input_file_id"]
-
-    async def test_validate_input_file_not_found(self, provider):
-        """Test _validate_input when input file does not exist."""
-        provider.files_api.openai_retrieve_file = AsyncMock(side_effect=Exception("File not found"))
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id="nonexistent_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 1
-        assert len(requests) == 0
-        assert errors[0].code == "invalid_request"
-        assert errors[0].message == "Cannot find file nonexistent_file."
-        assert errors[0].param == "input_file_id"
-        assert errors[0].line is None
-
-    async def test_validate_input_file_exists_empty_content(self, provider):
-        """Test _validate_input when file exists but is empty."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-        mock_response.body = b""
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id="empty_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 0
-        assert len(requests) == 0
-
-    async def test_validate_input_file_mixed_valid_invalid_json(self, provider):
-        """Test _validate_input when file contains valid and invalid JSON lines."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-        # Line 1: valid JSON with proper body args, Line 2: invalid JSON
-        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]}}\n{invalid json'
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id="mixed_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        # Should have 1 JSON parsing error from line 2, and 1 valid request from line 1
-        assert len(errors) == 1
-        assert len(requests) == 1
-
-        assert errors[0].code == "invalid_json_line"
-        assert errors[0].line == 2
-        assert errors[0].message == "This line is not parseable as valid JSON."
-
-        assert requests[0].custom_id == "req-1"
-        assert requests[0].method == "POST"
-        assert requests[0].url == "/v1/chat/completions"
-        assert requests[0].body["model"] == "test-model"
-        assert requests[0].body["messages"] == [{"role": "user", "content": "Hello"}]
-
-    async def test_validate_input_invalid_model(self, provider):
-        """Test _validate_input when file contains request with non-existent model."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "nonexistent-model", "messages": [{"role": "user", "content": "Hello"}]}}'
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        provider.models_api.get_model = AsyncMock(side_effect=Exception("Model not found"))
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id="invalid_model_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 1
-        assert len(requests) == 0
-
-        assert errors[0].code == "model_not_found"
-        assert errors[0].line == 1
-        assert errors[0].message == "Model 'nonexistent-model' does not exist or is not supported"
-        assert errors[0].param == "body.model"
-
-    @pytest.mark.parametrize(
-        "param_name,param_path,error_code,error_message",
-        [
-            ("custom_id", "custom_id", "missing_required_parameter", "Missing required parameter: custom_id"),
-            ("method", "method", "missing_required_parameter", "Missing required parameter: method"),
-            ("url", "url", "missing_required_parameter", "Missing required parameter: url"),
-            ("body", "body", "missing_required_parameter", "Missing required parameter: body"),
-            ("model", "body.model", "invalid_request", "Model parameter is required"),
-            ("messages", "body.messages", "invalid_request", "Messages parameter is required"),
-        ],
-    )
-    async def test_validate_input_missing_parameters(self, provider, param_name, param_path, error_code, error_message):
-        """Test _validate_input when file contains request with missing required parameters."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-
-        base_request = {
-            "custom_id": "req-1",
-            "method": "POST",
-            "url": "/v1/chat/completions",
-            "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]},
-        }
-
-        # Remove the specific parameter being tested
-        if "." in param_path:
-            top_level, nested_param = param_path.split(".", 1)
-            del base_request[top_level][nested_param]
-        else:
-            del base_request[param_name]
-
-        mock_response.body = json.dumps(base_request).encode()
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id=f"missing_{param_name}_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 1
-        assert len(requests) == 0
-
-        assert errors[0].code == error_code
-        assert errors[0].line == 1
-        assert errors[0].message == error_message
-        assert errors[0].param == param_path
-
-    async def test_validate_input_url_mismatch(self, provider):
-        """Test _validate_input when file contains request with URL that doesn't match batch endpoint."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/embeddings", "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]}}'
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",  # This doesn't match the URL in the request
-            input_file_id="url_mismatch_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 1
-        assert len(requests) == 0
-
-        assert errors[0].code == "invalid_url"
-        assert errors[0].line == 1
-        assert errors[0].message == "URL provided for this request does not match the batch endpoint"
-        assert errors[0].param == "url"
-
-    async def test_validate_input_multiple_errors_per_request(self, provider):
-        """Test _validate_input when a single request has multiple validation errors."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-        # Request missing custom_id, has invalid URL, and missing model in body
-        mock_response.body = (
-            b'{"method": "POST", "url": "/v1/embeddings", "body": {"messages": [{"role": "user", "content": "Hello"}]}}'
-        )
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",  # Doesn't match /v1/embeddings in request
-            input_file_id="multiple_errors_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) >= 2  # At least missing custom_id and URL mismatch
-        assert len(requests) == 0
-
-        for error in errors:
-            assert error.line == 1
-
-        error_codes = {error.code for error in errors}
-        assert "missing_required_parameter" in error_codes  # missing custom_id
-        assert "invalid_url" in error_codes  # URL mismatch
-
-    async def test_validate_input_invalid_request_format(self, provider):
-        """Test _validate_input when file contains non-object JSON (array, string, number)."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-        mock_response.body = b'["not", "a", "request", "object"]'
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id="invalid_format_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 1
-        assert len(requests) == 0
-
-        assert errors[0].code == "invalid_request"
-        assert errors[0].line == 1
-        assert errors[0].message == "Each line must be a JSON dictionary object"
-
-    @pytest.mark.parametrize(
-        "param_name,param_path,invalid_value,error_message",
-        [
-            ("custom_id", "custom_id", 12345, "Custom_id must be a string"),
-            ("url", "url", 123, "URL must be a string"),
-            ("method", "method", ["POST"], "Method must be a string"),
-            ("body", "body", ["not", "valid"], "Body must be a JSON dictionary object"),
-            ("model", "body.model", 123, "Model must be a string"),
-            ("messages", "body.messages", "invalid messages format", "Messages must be an array"),
-        ],
-    )
-    async def test_validate_input_invalid_parameter_types(
-        self, provider, param_name, param_path, invalid_value, error_message
-    ):
-        """Test _validate_input when file contains request with parameters that have invalid types."""
-        provider.files_api.openai_retrieve_file = AsyncMock()
-        mock_response = MagicMock()
-
-        base_request = {
-            "custom_id": "req-1",
-            "method": "POST",
-            "url": "/v1/chat/completions",
-            "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]},
-        }
-
-        # Override the specific parameter with invalid value
-        if "." in param_path:
-            top_level, nested_param = param_path.split(".", 1)
-            base_request[top_level][nested_param] = invalid_value
-        else:
-            base_request[param_name] = invalid_value
-
-        mock_response.body = json.dumps(base_request).encode()
-        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
-
-        batch = BatchObject(
-            id="batch_test",
-            object="batch",
-            endpoint="/v1/chat/completions",
-            input_file_id=f"invalid_{param_name}_type_file",
-            completion_window="24h",
-            status="validating",
-            created_at=1234567890,
-        )
-
-        errors, requests = await provider._validate_input(batch)
-
-        assert len(errors) == 1
-        assert len(requests) == 0
-
-        assert errors[0].code == "invalid_request"
-        assert errors[0].line == 1
-        assert errors[0].message == error_message
-        assert errors[0].param == param_path
-
-    async def test_max_concurrent_batches(self, provider):
-        """Test max_concurrent_batches configuration and concurrency control."""
-        import asyncio
-
-        provider._batch_semaphore = asyncio.Semaphore(2)
-
-        provider.process_batches = True  # enable because we're testing background processing
-
-        active_batches = 0
-
-        async def add_and_wait(batch_id: str):
-            nonlocal active_batches
-            active_batches += 1
-            await asyncio.sleep(float("inf"))
-
-        # the first thing done in _process_batch is to acquire the semaphore, then call _process_batch_impl,
-        # so we can replace _process_batch_impl with our mock to control concurrency
-        provider._process_batch_impl = add_and_wait
-
-        for _ in range(3):
-            await provider.create_batch(
-                input_file_id="file_id", endpoint="/v1/chat/completions", completion_window="24h"
-            )
-
-        await asyncio.sleep(0.042)  # let tasks start
-
-        assert active_batches == 2, f"Expected 2 active batches, got {active_batches}"

From c15cc7ed77b7689e9fdf24cbda12a4511db21f89 Mon Sep 17 00:00:00 2001
From: Derek Higgins <derekh@redhat.com>
Date: Thu, 14 Aug 2025 18:27:00 +0100
Subject: [PATCH 27/45] fix: use ChatCompletionMessageFunctionToolCall (#3142)

The OpenAI compatibility layer was incorrectly importing
ChatCompletionMessageToolCallParam instead of the
ChatCompletionMessageFunctionToolCall class. This caused "Cannot
instantiate typing.Union" errors when processing agent requests with
tool calls.

Closes: #3141

Signed-off-by: Derek Higgins <derekh@redhat.com>
---
 .../utils/inference/openai_compat.py          | 10 ++---
 .../utils/inference/test_openai_compat.py     | 40 +++++++++++++++++++
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py
index 9a77c8cc4..6297cc2ed 100644
--- a/llama_stack/providers/utils/inference/openai_compat.py
+++ b/llama_stack/providers/utils/inference/openai_compat.py
@@ -31,15 +31,15 @@ from openai.types.chat import (
 from openai.types.chat import (
     ChatCompletionContentPartTextParam as OpenAIChatCompletionContentPartTextParam,
 )
+from openai.types.chat import (
+    ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall,
+)
 from openai.types.chat import (
     ChatCompletionMessageParam as OpenAIChatCompletionMessage,
 )
 from openai.types.chat import (
     ChatCompletionMessageToolCall,
 )
-from openai.types.chat import (
-    ChatCompletionMessageToolCallParam as OpenAIChatCompletionMessageToolCall,
-)
 from openai.types.chat import (
     ChatCompletionSystemMessageParam as OpenAIChatCompletionSystemMessage,
 )
@@ -633,7 +633,7 @@ async def convert_message_to_openai_dict_new(
         )
     elif isinstance(message, CompletionMessage):
         tool_calls = [
-            OpenAIChatCompletionMessageToolCall(
+            OpenAIChatCompletionMessageFunctionToolCall(
                 id=tool.call_id,
                 function=OpenAIFunction(
                     name=(tool.tool_name if not isinstance(tool.tool_name, BuiltinTool) else tool.tool_name.value),
@@ -903,7 +903,7 @@ def _convert_openai_request_response_format(
 
 
 def _convert_openai_tool_calls(
-    tool_calls: list[OpenAIChatCompletionMessageToolCall],
+    tool_calls: list[OpenAIChatCompletionMessageFunctionToolCall],
 ) -> list[ToolCall]:
     """
     Convert an OpenAI ChatCompletionMessageToolCall list into a list of ToolCall.
diff --git a/tests/unit/providers/utils/inference/test_openai_compat.py b/tests/unit/providers/utils/inference/test_openai_compat.py
index 5b8527d1b..ddc70e102 100644
--- a/tests/unit/providers/utils/inference/test_openai_compat.py
+++ b/tests/unit/providers/utils/inference/test_openai_compat.py
@@ -24,6 +24,7 @@ from llama_stack.apis.inference import (
 from llama_stack.models.llama.datatypes import BuiltinTool, StopReason, ToolCall
 from llama_stack.providers.utils.inference.openai_compat import (
     convert_message_to_openai_dict,
+    convert_message_to_openai_dict_new,
     openai_messages_to_messages,
 )
 
@@ -182,3 +183,42 @@ def test_user_message_accepts_images():
     assert len(msg.content) == 2
     assert msg.content[0].text == "Describe this image:"
     assert msg.content[1].image_url.url == "http://example.com/image.jpg"
+
+
+async def test_convert_message_to_openai_dict_new_user_message():
+    """Test convert_message_to_openai_dict_new with UserMessage."""
+    message = UserMessage(content="Hello, world!", role="user")
+    result = await convert_message_to_openai_dict_new(message)
+
+    assert result["role"] == "user"
+    assert result["content"] == "Hello, world!"
+
+
+async def test_convert_message_to_openai_dict_new_completion_message_with_tool_calls():
+    """Test convert_message_to_openai_dict_new with CompletionMessage containing tool calls."""
+    message = CompletionMessage(
+        content="I'll help you find the weather.",
+        tool_calls=[
+            ToolCall(
+                call_id="call_123",
+                tool_name="get_weather",
+                arguments={"city": "Sligo"},
+                arguments_json='{"city": "Sligo"}',
+            )
+        ],
+        stop_reason=StopReason.end_of_turn,
+    )
+    result = await convert_message_to_openai_dict_new(message)
+
+    # This would have failed with "Cannot instantiate typing.Union" before the fix
+    assert result["role"] == "assistant"
+    assert result["content"] == "I'll help you find the weather."
+    assert "tool_calls" in result
+    assert result["tool_calls"] is not None
+    assert len(result["tool_calls"]) == 1
+
+    tool_call = result["tool_calls"][0]
+    assert tool_call.id == "call_123"
+    assert tool_call.type == "function"
+    assert tool_call.function.name == "get_weather"
+    assert tool_call.function.arguments == '{"city": "Sligo"}'

From 61582f327cc44dad9e79b1f06e8fba516832908d Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Thu, 14 Aug 2025 10:27:25 -0700
Subject: [PATCH 28/45] fix(ci): update triggers for the workflows (#3152)

---
 .github/workflows/integration-tests.yml        | 4 ++--
 .github/workflows/record-integration-tests.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index f330d2c45..9ef49fba3 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -5,7 +5,7 @@ run-name: Run the integration test suite from tests/integration in replay mode
 on:
   push:
     branches: [ main ]
-  pull_request_target:
+  pull_request:
     branches: [ main ]
     types: [opened, synchronize, reopened]
     paths:
@@ -34,7 +34,7 @@ on:
 
 concurrency:
   # Skip concurrency for pushes to main - each commit should be tested independently
-  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number }}
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
   cancel-in-progress: true
 
 jobs:
diff --git a/.github/workflows/record-integration-tests.yml b/.github/workflows/record-integration-tests.yml
index 12957db27..b31709a4f 100644
--- a/.github/workflows/record-integration-tests.yml
+++ b/.github/workflows/record-integration-tests.yml
@@ -3,7 +3,7 @@ name: Integration Tests (Record)
 run-name: Run the integration test suite from tests/integration
 
 on:
-  pull_request:
+  pull_request_target:
     branches: [ main ]
     types: [opened, synchronize, labeled]
     paths:
@@ -23,7 +23,7 @@ on:
         default: 'ollama'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 jobs:

From e69acbafbfd902333ede09361c214fc82a8e895a Mon Sep 17 00:00:00 2001
From: Francisco Arceo <arceofrancisco@gmail.com>
Date: Thu, 14 Aug 2025 15:58:43 -0600
Subject: [PATCH 29/45] feat(UI): Adding linter and prettier for UI (#3156)

---
 .pre-commit-config.yaml                       |  15 ++
 llama_stack/ui/.nvmrc                         |   1 +
 llama_stack/ui/.prettierignore                |   9 +
 llama_stack/ui/.prettierrc                    |  11 +-
 llama_stack/ui/app/api/v1/[...path]/route.ts  |   4 +-
 llama_stack/ui/app/auth/signin/page.tsx       |   4 +-
 llama_stack/ui/app/chat-playground/page.tsx   | 192 ++++++++------
 .../app/logs/chat-completions/[id]/page.tsx   |   4 +-
 .../ui/app/logs/responses/[id]/page.tsx       |  16 +-
 .../[fileId]/contents/[contentId]/page.tsx    | 133 ++++++----
 .../[id]/files/[fileId]/contents/page.tsx     |  88 +++++--
 .../[id]/files/[fileId]/page.tsx              |  90 +++++--
 .../ui/app/logs/vector-stores/[id]/page.tsx   |  15 +-
 .../ui/app/logs/vector-stores/page.tsx        | 137 +++++-----
 .../chat-completion-detail.test.tsx           |  28 +--
 .../chat-completion-detail.tsx                |  29 ++-
 .../chat-completion-table.test.tsx            |  34 +--
 .../chat-completions-table.tsx                |   5 +-
 .../chat-completions/chat-messasge-item.tsx   |  35 +--
 .../chat-playground/chat-message.tsx          | 156 ++++++------
 .../ui/components/chat-playground/chat.tsx    | 211 ++++++++--------
 .../chat-playground/interrupt-prompt.tsx      |  12 +-
 .../chat-playground/markdown-renderer.tsx     | 120 ++++-----
 .../chat-playground/message-input.tsx         | 237 +++++++++---------
 .../chat-playground/message-list.tsx          |  20 +-
 .../chat-playground/prompt-suggestions.tsx    |  10 +-
 .../chat-playground/typing-indicator.tsx      |   4 +-
 .../ui/components/layout/app-sidebar.tsx      | 103 ++++----
 .../ui/components/layout/detail-layout.tsx    |   2 +-
 .../logs/logs-table-scroll.test.tsx           |  22 +-
 .../ui/components/logs/logs-table.test.tsx    |  42 ++--
 llama_stack/ui/components/logs/logs-table.tsx |   2 +-
 .../grouping/grouped-items-display.tsx        |   2 +-
 .../responses/hooks/function-call-grouping.ts |   2 +-
 .../responses/items/item-renderer.tsx         |   2 +-
 .../responses/items/message-item.tsx          |   2 +-
 .../responses/responses-detail.test.tsx       |  60 ++---
 .../responses/responses-table.test.tsx        |  34 +--
 .../components/responses/responses-table.tsx  |  20 +-
 .../components/responses/utils/item-types.ts  |  10 +-
 .../ui/components/ui/audio-visualizer.tsx     | 146 +++++------
 llama_stack/ui/components/ui/breadcrumb.tsx   |   2 +-
 llama_stack/ui/components/ui/button.tsx       |  18 +-
 llama_stack/ui/components/ui/card.tsx         |   6 +-
 llama_stack/ui/components/ui/collapsible.tsx  |  12 +-
 llama_stack/ui/components/ui/copy-button.tsx  |  20 +-
 .../ui/components/ui/dropdown-menu.tsx        |  16 +-
 llama_stack/ui/components/ui/file-preview.tsx |  56 ++---
 llama_stack/ui/components/ui/input.tsx        |   2 +-
 llama_stack/ui/components/ui/select.tsx       |  34 +--
 llama_stack/ui/components/ui/separator.tsx    |   2 +-
 llama_stack/ui/components/ui/sheet.tsx        |   4 +-
 llama_stack/ui/components/ui/sidebar.tsx      |  36 +--
 llama_stack/ui/components/ui/sonner.tsx       |  14 +-
 llama_stack/ui/components/ui/table.tsx        |   8 +-
 llama_stack/ui/components/ui/tooltip.tsx      |   2 +-
 .../vector-stores/vector-store-detail.tsx     |   4 +-
 llama_stack/ui/e2e/logs-table-scroll.spec.ts  |   2 +-
 llama_stack/ui/eslint.config.mjs              |   8 +-
 llama_stack/ui/hooks/use-audio-recording.ts   |  82 +++---
 llama_stack/ui/hooks/use-auto-scroll.ts       |  50 ++--
 llama_stack/ui/hooks/use-autosize-textarea.ts |  32 +--
 llama_stack/ui/hooks/use-copy-to-clipboard.ts |  34 +--
 llama_stack/ui/hooks/use-infinite-scroll.ts   |   6 +-
 llama_stack/ui/hooks/use-mobile.ts            |   2 +-
 llama_stack/ui/hooks/use-pagination.ts        |  10 +-
 llama_stack/ui/lib/audio-utils.ts             |  54 ++--
 llama_stack/ui/lib/config-validator.ts        |   8 +-
 llama_stack/ui/lib/contents-api.ts            |  45 ++--
 .../ui/lib/format-message-content.test.ts     |  28 ++-
 llama_stack/ui/lib/format-message-content.ts  |   4 +-
 llama_stack/ui/lib/format-tool-call.tsx       |   6 +-
 llama_stack/ui/lib/truncate-text.ts           |   2 +-
 73 files changed, 1452 insertions(+), 1226 deletions(-)
 create mode 100644 llama_stack/ui/.nvmrc

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 30843173c..4309f289a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,6 +2,7 @@ exclude: 'build/'
 
 default_language_version:
     python: python3.12
+    node: "22"
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
@@ -145,6 +146,20 @@ repos:
         pass_filenames: false
         require_serial: true
         files: ^.github/workflows/.*$
+      - id: ui-prettier
+        name: Format UI code with Prettier
+        entry: bash -c 'cd llama_stack/ui && npm run format'
+        language: system
+        files: ^llama_stack/ui/.*\.(ts|tsx)$
+        pass_filenames: false
+        require_serial: true
+      - id: ui-eslint
+        name: Lint UI code with ESLint
+        entry: bash -c 'cd llama_stack/ui && npm run lint -- --fix --quiet'
+        language: system
+        files: ^llama_stack/ui/.*\.(ts|tsx)$
+        pass_filenames: false
+        require_serial: true
 
 ci:
     autofix_commit_msg: 🎨 [pre-commit.ci] Auto format from pre-commit.com hooks
diff --git a/llama_stack/ui/.nvmrc b/llama_stack/ui/.nvmrc
new file mode 100644
index 000000000..1384ff6a1
--- /dev/null
+++ b/llama_stack/ui/.nvmrc
@@ -0,0 +1 @@
+22.5.1
diff --git a/llama_stack/ui/.prettierignore b/llama_stack/ui/.prettierignore
index 1b8ac8894..b737ae6ed 100644
--- a/llama_stack/ui/.prettierignore
+++ b/llama_stack/ui/.prettierignore
@@ -1,3 +1,12 @@
 # Ignore artifacts:
 build
 coverage
+.next
+node_modules
+dist
+*.lock
+*.log
+
+# Generated files
+*.min.js
+*.min.css
diff --git a/llama_stack/ui/.prettierrc b/llama_stack/ui/.prettierrc
index 0967ef424..059475a24 100644
--- a/llama_stack/ui/.prettierrc
+++ b/llama_stack/ui/.prettierrc
@@ -1 +1,10 @@
-{}
+{
+  "semi": true,
+  "trailingComma": "es5",
+  "singleQuote": false,
+  "printWidth": 80,
+  "tabWidth": 2,
+  "useTabs": false,
+  "bracketSpacing": true,
+  "arrowParens": "avoid"
+}
diff --git a/llama_stack/ui/app/api/v1/[...path]/route.ts b/llama_stack/ui/app/api/v1/[...path]/route.ts
index 1959f9099..51c1f8004 100644
--- a/llama_stack/ui/app/api/v1/[...path]/route.ts
+++ b/llama_stack/ui/app/api/v1/[...path]/route.ts
@@ -47,7 +47,7 @@ async function proxyRequest(request: NextRequest, method: string) {
     const responseText = await response.text();
 
     console.log(
-      `Response from FastAPI: ${response.status} ${response.statusText}`,
+      `Response from FastAPI: ${response.status} ${response.statusText}`
     );
 
     // Create response with same status and headers
@@ -74,7 +74,7 @@ async function proxyRequest(request: NextRequest, method: string) {
         backend_url: BACKEND_URL,
         timestamp: new Date().toISOString(),
       },
-      { status: 500 },
+      { status: 500 }
     );
   }
 }
diff --git a/llama_stack/ui/app/auth/signin/page.tsx b/llama_stack/ui/app/auth/signin/page.tsx
index c9510fd6b..0ccb4a397 100644
--- a/llama_stack/ui/app/auth/signin/page.tsx
+++ b/llama_stack/ui/app/auth/signin/page.tsx
@@ -51,9 +51,9 @@ export default function SignInPage() {
               onClick={() => {
                 console.log("Signing in with GitHub...");
                 signIn("github", { callbackUrl: "/auth/signin" }).catch(
-                  (error) => {
+                  error => {
                     console.error("Sign in error:", error);
-                  },
+                  }
                 );
               }}
               className="w-full"
diff --git a/llama_stack/ui/app/chat-playground/page.tsx b/llama_stack/ui/app/chat-playground/page.tsx
index d8094af85..b8651aca0 100644
--- a/llama_stack/ui/app/chat-playground/page.tsx
+++ b/llama_stack/ui/app/chat-playground/page.tsx
@@ -29,14 +29,13 @@ export default function ChatPlaygroundPage() {
 
   const isModelsLoading = modelsLoading ?? true;
 
-
   useEffect(() => {
     const fetchModels = async () => {
       try {
         setModelsLoading(true);
         setModelsError(null);
         const modelList = await client.models.list();
-        const llmModels = modelList.filter(model => model.model_type === 'llm');
+        const llmModels = modelList.filter(model => model.model_type === "llm");
         setModels(llmModels);
         if (llmModels.length > 0) {
           setSelectedModel(llmModels[0].identifier);
@@ -53,103 +52,122 @@ export default function ChatPlaygroundPage() {
   }, [client]);
 
   const extractTextContent = (content: unknown): string => {
-    if (typeof content === 'string') {
+    if (typeof content === "string") {
       return content;
     }
     if (Array.isArray(content)) {
       return content
-        .filter(item => item && typeof item === 'object' && 'type' in item && item.type === 'text')
-        .map(item => (item && typeof item === 'object' && 'text' in item) ? String(item.text) : '')
-        .join('');
+        .filter(
+          item =>
+            item &&
+            typeof item === "object" &&
+            "type" in item &&
+            item.type === "text"
+        )
+        .map(item =>
+          item && typeof item === "object" && "text" in item
+            ? String(item.text)
+            : ""
+        )
+        .join("");
     }
-    if (content && typeof content === 'object' && 'type' in content && content.type === 'text' && 'text' in content) {
-      return String(content.text) || '';
+    if (
+      content &&
+      typeof content === "object" &&
+      "type" in content &&
+      content.type === "text" &&
+      "text" in content
+    ) {
+      return String(content.text) || "";
     }
-    return '';
+    return "";
   };
 
   const handleInputChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
     setInput(e.target.value);
   };
 
-const handleSubmit = async (event?: { preventDefault?: () => void }) => {
-  event?.preventDefault?.();
-  if (!input.trim()) return;
+  const handleSubmit = async (event?: { preventDefault?: () => void }) => {
+    event?.preventDefault?.();
+    if (!input.trim()) return;
 
-  // Add user message to chat
-  const userMessage: Message = {
-    id: Date.now().toString(),
-    role: "user",
-    content: input.trim(),
-    createdAt: new Date(),
-  };
-
-  setMessages(prev => [...prev, userMessage]);
-  setInput("");
-
-  // Use the helper function with the content
-  await handleSubmitWithContent(userMessage.content);
-};
-
-const handleSubmitWithContent = async (content: string) => {
-  setIsGenerating(true);
-  setError(null);
-
-  try {
-    const messageParams: CompletionCreateParams["messages"] = [
-      ...messages.map(msg => {
-        const msgContent = typeof msg.content === 'string' ? msg.content : extractTextContent(msg.content);
-        if (msg.role === "user") {
-          return { role: "user" as const, content: msgContent };
-        } else if (msg.role === "assistant") {
-          return { role: "assistant" as const, content: msgContent };
-        } else {
-          return { role: "system" as const, content: msgContent };
-        }
-      }),
-      { role: "user" as const, content }
-    ];
-
-    const response = await client.chat.completions.create({
-      model: selectedModel,
-      messages: messageParams,
-      stream: true,
-    });
-
-    const assistantMessage: Message = {
-      id: (Date.now() + 1).toString(),
-      role: "assistant",
-      content: "",
+    // Add user message to chat
+    const userMessage: Message = {
+      id: Date.now().toString(),
+      role: "user",
+      content: input.trim(),
       createdAt: new Date(),
     };
 
-    setMessages(prev => [...prev, assistantMessage]);
-    let fullContent = "";
-    for await (const chunk of response) {
-      if (chunk.choices && chunk.choices[0]?.delta?.content) {
-        const deltaContent = chunk.choices[0].delta.content;
-        fullContent += deltaContent;
+    setMessages(prev => [...prev, userMessage]);
+    setInput("");
 
-        flushSync(() => {
-          setMessages(prev => {
-            const newMessages = [...prev];
-            const lastMessage = newMessages[newMessages.length - 1];
-            if (lastMessage.role === "assistant") {
-              lastMessage.content = fullContent;
-            }
-            return newMessages;
+    // Use the helper function with the content
+    await handleSubmitWithContent(userMessage.content);
+  };
+
+  const handleSubmitWithContent = async (content: string) => {
+    setIsGenerating(true);
+    setError(null);
+
+    try {
+      const messageParams: CompletionCreateParams["messages"] = [
+        ...messages.map(msg => {
+          const msgContent =
+            typeof msg.content === "string"
+              ? msg.content
+              : extractTextContent(msg.content);
+          if (msg.role === "user") {
+            return { role: "user" as const, content: msgContent };
+          } else if (msg.role === "assistant") {
+            return { role: "assistant" as const, content: msgContent };
+          } else {
+            return { role: "system" as const, content: msgContent };
+          }
+        }),
+        { role: "user" as const, content },
+      ];
+
+      const response = await client.chat.completions.create({
+        model: selectedModel,
+        messages: messageParams,
+        stream: true,
+      });
+
+      const assistantMessage: Message = {
+        id: (Date.now() + 1).toString(),
+        role: "assistant",
+        content: "",
+        createdAt: new Date(),
+      };
+
+      setMessages(prev => [...prev, assistantMessage]);
+      let fullContent = "";
+      for await (const chunk of response) {
+        if (chunk.choices && chunk.choices[0]?.delta?.content) {
+          const deltaContent = chunk.choices[0].delta.content;
+          fullContent += deltaContent;
+
+          flushSync(() => {
+            setMessages(prev => {
+              const newMessages = [...prev];
+              const lastMessage = newMessages[newMessages.length - 1];
+              if (lastMessage.role === "assistant") {
+                lastMessage.content = fullContent;
+              }
+              return newMessages;
+            });
           });
-        });
+        }
       }
+    } catch (err) {
+      console.error("Error sending message:", err);
+      setError("Failed to send message. Please try again.");
+      setMessages(prev => prev.slice(0, -1));
+    } finally {
+      setIsGenerating(false);
     }
-  } catch (err) {
-    console.error("Error sending message:", err);
-    setError("Failed to send message. Please try again.");
-    setMessages(prev => prev.slice(0, -1));
-  } finally {
-    setIsGenerating(false);
-  }
-};
+  };
   const suggestions = [
     "Write a Python function that prints 'Hello, World!'",
     "Explain step-by-step how to solve this math problem: If x² + 6x + 9 = 25, what is x?",
@@ -163,7 +181,7 @@ const handleSubmitWithContent = async (content: string) => {
       content: message.content,
       createdAt: new Date(),
     };
-    setMessages(prev => [...prev, newMessage])
+    setMessages(prev => [...prev, newMessage]);
     handleSubmitWithContent(newMessage.content);
   };
 
@@ -177,12 +195,20 @@ const handleSubmitWithContent = async (content: string) => {
       <div className="mb-4 flex justify-between items-center">
         <h1 className="text-2xl font-bold">Chat Playground (Completions)</h1>
         <div className="flex gap-2">
-          <Select value={selectedModel} onValueChange={setSelectedModel} disabled={isModelsLoading || isGenerating}>
+          <Select
+            value={selectedModel}
+            onValueChange={setSelectedModel}
+            disabled={isModelsLoading || isGenerating}
+          >
             <SelectTrigger className="w-[180px]">
-              <SelectValue placeholder={isModelsLoading ? "Loading models..." : "Select Model"} />
+              <SelectValue
+                placeholder={
+                  isModelsLoading ? "Loading models..." : "Select Model"
+                }
+              />
             </SelectTrigger>
             <SelectContent>
-              {models.map((model) => (
+              {models.map(model => (
                 <SelectItem key={model.identifier} value={model.identifier}>
                   {model.identifier}
                 </SelectItem>
diff --git a/llama_stack/ui/app/logs/chat-completions/[id]/page.tsx b/llama_stack/ui/app/logs/chat-completions/[id]/page.tsx
index 82aa3496e..e11924f4c 100644
--- a/llama_stack/ui/app/logs/chat-completions/[id]/page.tsx
+++ b/llama_stack/ui/app/logs/chat-completions/[id]/page.tsx
@@ -33,12 +33,12 @@ export default function ChatCompletionDetailPage() {
       } catch (err) {
         console.error(
           `Error fetching chat completion detail for ID ${id}:`,
-          err,
+          err
         );
         setError(
           err instanceof Error
             ? err
-            : new Error("Failed to fetch completion detail"),
+            : new Error("Failed to fetch completion detail")
         );
       } finally {
         setIsLoading(false);
diff --git a/llama_stack/ui/app/logs/responses/[id]/page.tsx b/llama_stack/ui/app/logs/responses/[id]/page.tsx
index 7f4252856..922d35531 100644
--- a/llama_stack/ui/app/logs/responses/[id]/page.tsx
+++ b/llama_stack/ui/app/logs/responses/[id]/page.tsx
@@ -13,10 +13,10 @@ export default function ResponseDetailPage() {
   const client = useAuthClient();
 
   const [responseDetail, setResponseDetail] = useState<OpenAIResponse | null>(
-    null,
+    null
   );
   const [inputItems, setInputItems] = useState<InputItemListResponse | null>(
-    null,
+    null
   );
   const [isLoading, setIsLoading] = useState<boolean>(true);
   const [isLoadingInputItems, setIsLoadingInputItems] = useState<boolean>(true);
@@ -25,7 +25,7 @@ export default function ResponseDetailPage() {
 
   // Helper function to convert ResponseObject to OpenAIResponse
   const convertResponseObject = (
-    responseData: ResponseObject,
+    responseData: ResponseObject
   ): OpenAIResponse => {
     return {
       id: responseData.id,
@@ -73,12 +73,12 @@ export default function ResponseDetailPage() {
         } else {
           console.error(
             `Error fetching response detail for ID ${id}:`,
-            responseResult.reason,
+            responseResult.reason
           );
           setError(
             responseResult.reason instanceof Error
               ? responseResult.reason
-              : new Error("Failed to fetch response detail"),
+              : new Error("Failed to fetch response detail")
           );
         }
 
@@ -90,18 +90,18 @@ export default function ResponseDetailPage() {
         } else {
           console.error(
             `Error fetching input items for response ID ${id}:`,
-            inputItemsResult.reason,
+            inputItemsResult.reason
           );
           setInputItemsError(
             inputItemsResult.reason instanceof Error
               ? inputItemsResult.reason
-              : new Error("Failed to fetch input items"),
+              : new Error("Failed to fetch input items")
           );
         }
       } catch (err) {
         console.error(`Unexpected error fetching data for ID ${id}:`, err);
         setError(
-          err instanceof Error ? err : new Error("Unexpected error occurred"),
+          err instanceof Error ? err : new Error("Unexpected error occurred")
         );
       } finally {
         setIsLoading(false);
diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.tsx
index 6896b992a..d58de3085 100644
--- a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.tsx
+++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.tsx
@@ -18,7 +18,10 @@ import {
   PropertiesCard,
   PropertyItem,
 } from "@/components/layout/detail-layout";
-import { PageBreadcrumb, BreadcrumbSegment } from "@/components/layout/page-breadcrumb";
+import {
+  PageBreadcrumb,
+  BreadcrumbSegment,
+} from "@/components/layout/page-breadcrumb";
 
 export default function ContentDetailPage() {
   const params = useParams();
@@ -28,13 +31,13 @@ export default function ContentDetailPage() {
   const contentId = params.contentId as string;
   const client = useAuthClient();
 
-  const getTextFromContent = (content: any): string => {
-    if (typeof content === 'string') {
+  const getTextFromContent = (content: unknown): string => {
+    if (typeof content === "string") {
       return content;
-    } else if (content && content.type === 'text') {
+    } else if (content && content.type === "text") {
       return content.text;
     }
-    return '';
+    return "";
   };
 
   const [store, setStore] = useState<VectorStore | null>(null);
@@ -44,7 +47,9 @@ export default function ContentDetailPage() {
   const [error, setError] = useState<Error | null>(null);
   const [isEditing, setIsEditing] = useState(false);
   const [editedContent, setEditedContent] = useState("");
-  const [editedMetadata, setEditedMetadata] = useState<Record<string, any>>({});
+  const [editedMetadata, setEditedMetadata] = useState<Record<string, unknown>>(
+    {}
+  );
   const [isEditingEmbedding, setIsEditingEmbedding] = useState(false);
   const [editedEmbedding, setEditedEmbedding] = useState<number[]>([]);
 
@@ -64,8 +69,13 @@ export default function ContentDetailPage() {
         setFile(fileResponse as VectorStoreFile);
 
         const contentsAPI = new ContentsAPI(client);
-        const contentsResponse = await contentsAPI.listContents(vectorStoreId, fileId);
-        const targetContent = contentsResponse.data.find(c => c.id === contentId);
+        const contentsResponse = await contentsAPI.listContents(
+          vectorStoreId,
+          fileId
+        );
+        const targetContent = contentsResponse.data.find(
+          c => c.id === contentId
+        );
 
         if (targetContent) {
           setContent(targetContent);
@@ -76,7 +86,9 @@ export default function ContentDetailPage() {
           throw new Error(`Content ${contentId} not found`);
         }
       } catch (err) {
-        setError(err instanceof Error ? err : new Error("Failed to load content."));
+        setError(
+          err instanceof Error ? err : new Error("Failed to load content.")
+        );
       } finally {
         setIsLoading(false);
       }
@@ -88,7 +100,8 @@ export default function ContentDetailPage() {
     if (!content) return;
 
     try {
-      const updates: { content?: string; metadata?: Record<string, any> } = {};
+      const updates: { content?: string; metadata?: Record<string, unknown> } =
+        {};
 
       if (editedContent !== getTextFromContent(content.content)) {
         updates.content = editedContent;
@@ -100,25 +113,32 @@ export default function ContentDetailPage() {
 
       if (Object.keys(updates).length > 0) {
         const contentsAPI = new ContentsAPI(client);
-        const updatedContent = await contentsAPI.updateContent(vectorStoreId, fileId, contentId, updates);
+        const updatedContent = await contentsAPI.updateContent(
+          vectorStoreId,
+          fileId,
+          contentId,
+          updates
+        );
         setContent(updatedContent);
       }
 
       setIsEditing(false);
     } catch (err) {
-      console.error('Failed to update content:', err);
+      console.error("Failed to update content:", err);
     }
   };
 
   const handleDelete = async () => {
-    if (!confirm('Are you sure you want to delete this content?')) return;
+    if (!confirm("Are you sure you want to delete this content?")) return;
 
     try {
       const contentsAPI = new ContentsAPI(client);
       await contentsAPI.deleteContent(vectorStoreId, fileId, contentId);
-      router.push(`/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents`);
+      router.push(
+        `/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents`
+      );
     } catch (err) {
-      console.error('Failed to delete content:', err);
+      console.error("Failed to delete content:", err);
     }
   };
 
@@ -134,10 +154,19 @@ export default function ContentDetailPage() {
 
   const breadcrumbSegments: BreadcrumbSegment[] = [
     { label: "Vector Stores", href: "/logs/vector-stores" },
-    { label: store?.name || vectorStoreId, href: `/logs/vector-stores/${vectorStoreId}` },
+    {
+      label: store?.name || vectorStoreId,
+      href: `/logs/vector-stores/${vectorStoreId}`,
+    },
     { label: "Files", href: `/logs/vector-stores/${vectorStoreId}` },
-    { label: fileId, href: `/logs/vector-stores/${vectorStoreId}/files/${fileId}` },
-    { label: "Contents", href: `/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents` },
+    {
+      label: fileId,
+      href: `/logs/vector-stores/${vectorStoreId}/files/${fileId}`,
+    },
+    {
+      label: "Contents",
+      href: `/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents`,
+    },
     { label: contentId },
   ];
 
@@ -186,7 +215,7 @@ export default function ContentDetailPage() {
           {isEditing ? (
             <textarea
               value={editedContent}
-              onChange={(e) => setEditedContent(e.target.value)}
+              onChange={e => setEditedContent(e.target.value)}
               className="w-full h-64 p-3 border rounded-md resize-none font-mono text-sm"
               placeholder="Enter content..."
             />
@@ -206,16 +235,23 @@ export default function ContentDetailPage() {
           <div className="flex gap-2">
             {isEditingEmbedding ? (
               <>
-                <Button size="sm" onClick={() => {
-                  setIsEditingEmbedding(false);
-                }}>
+                <Button
+                  size="sm"
+                  onClick={() => {
+                    setIsEditingEmbedding(false);
+                  }}
+                >
                   <Save className="h-4 w-4 mr-1" />
                   Save
                 </Button>
-                <Button size="sm" variant="outline" onClick={() => {
-                  setEditedEmbedding(content?.embedding || []);
-                  setIsEditingEmbedding(false);
-                }}>
+                <Button
+                  size="sm"
+                  variant="outline"
+                  onClick={() => {
+                    setEditedEmbedding(content?.embedding || []);
+                    setIsEditingEmbedding(false);
+                  }}
+                >
                   <X className="h-4 w-4 mr-1" />
                   Cancel
                 </Button>
@@ -237,14 +273,16 @@ export default function ContentDetailPage() {
                 </p>
                 <textarea
                   value={JSON.stringify(editedEmbedding, null, 2)}
-                  onChange={(e) => {
+                  onChange={e => {
                     try {
                       const parsed = JSON.parse(e.target.value);
-                      if (Array.isArray(parsed) && parsed.every(v => typeof v === 'number')) {
+                      if (
+                        Array.isArray(parsed) &&
+                        parsed.every(v => typeof v === "number")
+                      ) {
                         setEditedEmbedding(parsed);
                       }
-                    } catch {
-                    }
+                    } catch {}
                   }}
                   className="w-full h-32 p-3 border rounded-md resize-none font-mono text-xs"
                   placeholder="Enter embedding as JSON array..."
@@ -259,8 +297,15 @@ export default function ContentDetailPage() {
                 </div>
                 <div className="p-3 bg-gray-50 dark:bg-gray-800 rounded-md max-h-32 overflow-y-auto">
                   <pre className="whitespace-pre-wrap font-mono text-xs text-gray-900 dark:text-gray-100">
-                    [{content.embedding.slice(0, 20).map(v => v.toFixed(6)).join(', ')}
-                    {content.embedding.length > 20 ? `\n... and ${content.embedding.length - 20} more values` : ''}]
+                    [
+                    {content.embedding
+                      .slice(0, 20)
+                      .map(v => v.toFixed(6))
+                      .join(", ")}
+                    {content.embedding.length > 20
+                      ? `\n... and ${content.embedding.length - 20} more values`
+                      : ""}
+                    ]
                   </pre>
                 </div>
               </div>
@@ -284,7 +329,7 @@ export default function ContentDetailPage() {
                 <div key={key} className="flex gap-2">
                   <Input
                     value={key}
-                    onChange={(e) => {
+                    onChange={e => {
                       const newMetadata = { ...editedMetadata };
                       delete newMetadata[key];
                       newMetadata[e.target.value] = value;
@@ -294,11 +339,13 @@ export default function ContentDetailPage() {
                     className="flex-1"
                   />
                   <Input
-                    value={typeof value === 'string' ? value : JSON.stringify(value)}
-                    onChange={(e) => {
+                    value={
+                      typeof value === "string" ? value : JSON.stringify(value)
+                    }
+                    onChange={e => {
                       setEditedMetadata({
                         ...editedMetadata,
-                        [key]: e.target.value
+                        [key]: e.target.value,
                       });
                     }}
                     placeholder="Value"
@@ -312,7 +359,7 @@ export default function ContentDetailPage() {
                 onClick={() => {
                   setEditedMetadata({
                     ...editedMetadata,
-                    ['']: ''
+                    [""]: "",
                   });
                 }}
               >
@@ -325,7 +372,7 @@ export default function ContentDetailPage() {
                 <div key={key} className="flex justify-between py-1">
                   <span className="font-medium text-gray-600">{key}:</span>
                   <span className="font-mono text-sm">
-                    {typeof value === 'string' ? value : JSON.stringify(value)}
+                    {typeof value === "string" ? value : JSON.stringify(value)}
                   </span>
                 </div>
               ))}
@@ -351,15 +398,15 @@ export default function ContentDetailPage() {
         value={`${getTextFromContent(content.content).length} chars`}
       />
       {content.metadata.chunk_window && (
-        <PropertyItem
-          label="Position"
-          value={content.metadata.chunk_window}
-        />
+        <PropertyItem label="Position" value={content.metadata.chunk_window} />
       )}
       {file && (
         <>
           <PropertyItem label="File Status" value={file.status} />
-          <PropertyItem label="File Usage" value={`${file.usage_bytes} bytes`} />
+          <PropertyItem
+            label="File Usage"
+            value={`${file.usage_bytes} bytes`}
+          />
         </>
       )}
       {store && (
diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx
index d43223c6c..0283db9e7 100644
--- a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx
+++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx
@@ -18,7 +18,10 @@ import {
   PropertiesCard,
   PropertyItem,
 } from "@/components/layout/detail-layout";
-import { PageBreadcrumb, BreadcrumbSegment } from "@/components/layout/page-breadcrumb";
+import {
+  PageBreadcrumb,
+  BreadcrumbSegment,
+} from "@/components/layout/page-breadcrumb";
 import {
   Table,
   TableBody,
@@ -36,23 +39,21 @@ export default function ContentsListPage() {
   const fileId = params.fileId as string;
   const client = useAuthClient();
 
-  const getTextFromContent = (content: any): string => {
-    if (typeof content === 'string') {
+  const getTextFromContent = (content: unknown): string => {
+    if (typeof content === "string") {
       return content;
-    } else if (content && content.type === 'text') {
+    } else if (content && content.type === "text") {
       return content.text;
     }
-    return '';
+    return "";
   };
 
   const [store, setStore] = useState<VectorStore | null>(null);
   const [file, setFile] = useState<VectorStoreFile | null>(null);
   const [contents, setContents] = useState<VectorStoreContentItem[]>([]);
   const [isLoadingStore, setIsLoadingStore] = useState(true);
-  const [isLoadingFile, setIsLoadingFile] = useState(true);
   const [isLoadingContents, setIsLoadingContents] = useState(true);
   const [errorStore, setErrorStore] = useState<Error | null>(null);
-  const [errorFile, setErrorFile] = useState<Error | null>(null);
   const [errorContents, setErrorContents] = useState<Error | null>(null);
 
   useEffect(() => {
@@ -65,7 +66,9 @@ export default function ContentsListPage() {
         const response = await client.vectorStores.retrieve(vectorStoreId);
         setStore(response as VectorStore);
       } catch (err) {
-        setErrorStore(err instanceof Error ? err : new Error("Failed to load vector store."));
+        setErrorStore(
+          err instanceof Error ? err : new Error("Failed to load vector store.")
+        );
       } finally {
         setIsLoadingStore(false);
       }
@@ -80,10 +83,15 @@ export default function ContentsListPage() {
       setIsLoadingFile(true);
       setErrorFile(null);
       try {
-        const response = await client.vectorStores.files.retrieve(vectorStoreId, fileId);
+        const response = await client.vectorStores.files.retrieve(
+          vectorStoreId,
+          fileId
+        );
         setFile(response as VectorStoreFile);
       } catch (err) {
-        setErrorFile(err instanceof Error ? err : new Error("Failed to load file."));
+        setErrorFile(
+          err instanceof Error ? err : new Error("Failed to load file.")
+        );
       } finally {
         setIsLoadingFile(false);
       }
@@ -99,10 +107,16 @@ export default function ContentsListPage() {
       setErrorContents(null);
       try {
         const contentsAPI = new ContentsAPI(client);
-        const contentsResponse = await contentsAPI.listContents(vectorStoreId, fileId, { limit: 100 });
+        const contentsResponse = await contentsAPI.listContents(
+          vectorStoreId,
+          fileId,
+          { limit: 100 }
+        );
         setContents(contentsResponse.data);
       } catch (err) {
-        setErrorContents(err instanceof Error ? err : new Error("Failed to load contents."));
+        setErrorContents(
+          err instanceof Error ? err : new Error("Failed to load contents.")
+        );
       } finally {
         setIsLoadingContents(false);
       }
@@ -116,26 +130,36 @@ export default function ContentsListPage() {
       await contentsAPI.deleteContent(vectorStoreId, fileId, contentId);
       setContents(contents.filter(content => content.id !== contentId));
     } catch (err) {
-      console.error('Failed to delete content:', err);
+      console.error("Failed to delete content:", err);
     }
   };
 
   const handleViewContent = (contentId: string) => {
-    router.push(`/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents/${contentId}`);
+    router.push(
+      `/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents/${contentId}`
+    );
   };
 
   const title = `Contents in File: ${fileId}`;
 
   const breadcrumbSegments: BreadcrumbSegment[] = [
     { label: "Vector Stores", href: "/logs/vector-stores" },
-    { label: store?.name || vectorStoreId, href: `/logs/vector-stores/${vectorStoreId}` },
+    {
+      label: store?.name || vectorStoreId,
+      href: `/logs/vector-stores/${vectorStoreId}`,
+    },
     { label: "Files", href: `/logs/vector-stores/${vectorStoreId}` },
-    { label: fileId, href: `/logs/vector-stores/${vectorStoreId}/files/${fileId}` },
+    {
+      label: fileId,
+      href: `/logs/vector-stores/${vectorStoreId}/files/${fileId}`,
+    },
     { label: "Contents" },
   ];
 
   if (errorStore) {
-    return <DetailErrorView title={title} id={vectorStoreId} error={errorStore} />;
+    return (
+      <DetailErrorView title={title} id={vectorStoreId} error={errorStore} />
+    );
   }
   if (isLoadingStore) {
     return <DetailLoadingView title={title} />;
@@ -175,7 +199,7 @@ export default function ContentsListPage() {
                 </TableRow>
               </TableHeader>
               <TableBody>
-                {contents.map((content) => (
+                {contents.map(content => (
                   <TableRow key={content.id}>
                     <TableCell className="font-mono text-xs">
                       <Button
@@ -189,7 +213,10 @@ export default function ContentsListPage() {
                     </TableCell>
                     <TableCell>
                       <div className="max-w-md">
-                        <p className="text-sm truncate" title={getTextFromContent(content.content)}>
+                        <p
+                          className="text-sm truncate"
+                          title={getTextFromContent(content.content)}
+                        >
                           {getTextFromContent(content.content)}
                         </p>
                       </div>
@@ -197,12 +224,25 @@ export default function ContentsListPage() {
                     <TableCell className="text-xs text-gray-500">
                       {content.embedding && content.embedding.length > 0 ? (
                         <div className="max-w-xs">
-                          <span className="font-mono text-xs bg-gray-100 dark:bg-gray-800 rounded px-1 py-0.5" title={`${content.embedding.length}D vector: [${content.embedding.slice(0, 3).map(v => v.toFixed(3)).join(', ')}...]`}>
-                            [{content.embedding.slice(0, 3).map(v => v.toFixed(3)).join(', ')}...] ({content.embedding.length}D)
+                          <span
+                            className="font-mono text-xs bg-gray-100 dark:bg-gray-800 rounded px-1 py-0.5"
+                            title={`${content.embedding.length}D vector: [${content.embedding
+                              .slice(0, 3)
+                              .map(v => v.toFixed(3))
+                              .join(", ")}...]`}
+                          >
+                            [
+                            {content.embedding
+                              .slice(0, 3)
+                              .map(v => v.toFixed(3))
+                              .join(", ")}
+                            ...] ({content.embedding.length}D)
                           </span>
                         </div>
                       ) : (
-                        <span className="text-gray-400 dark:text-gray-500 italic">No embedding</span>
+                        <span className="text-gray-400 dark:text-gray-500 italic">
+                          No embedding
+                        </span>
                       )}
                     </TableCell>
                     <TableCell className="text-xs text-gray-500">
@@ -211,7 +251,9 @@ export default function ContentsListPage() {
                         : `${content.metadata.content_length || 0} chars`}
                     </TableCell>
                     <TableCell className="text-xs">
-                      {new Date(content.created_timestamp * 1000).toLocaleString()}
+                      {new Date(
+                        content.created_timestamp * 1000
+                      ).toLocaleString()}
                     </TableCell>
                     <TableCell>
                       <div className="flex gap-1">
diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.tsx
index bdcf76e1b..fc6ee43f5 100644
--- a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.tsx
+++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.tsx
@@ -4,9 +4,12 @@ import { useEffect, useState } from "react";
 import { useParams, useRouter } from "next/navigation";
 import { useAuthClient } from "@/hooks/use-auth-client";
 import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores";
-import type { VectorStoreFile, FileContentResponse } from "llama-stack-client/resources/vector-stores/files";
+import type {
+  VectorStoreFile,
+  FileContentResponse,
+} from "llama-stack-client/resources/vector-stores/files";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
-import { Skeleton } from '@/components/ui/skeleton';
+import { Skeleton } from "@/components/ui/skeleton";
 import { Button } from "@/components/ui/button";
 import { List } from "lucide-react";
 import {
@@ -17,7 +20,10 @@ import {
   PropertiesCard,
   PropertyItem,
 } from "@/components/layout/detail-layout";
-import { PageBreadcrumb, BreadcrumbSegment } from "@/components/layout/page-breadcrumb";
+import {
+  PageBreadcrumb,
+  BreadcrumbSegment,
+} from "@/components/layout/page-breadcrumb";
 
 export default function FileDetailPage() {
   const params = useParams();
@@ -46,7 +52,9 @@ export default function FileDetailPage() {
         const response = await client.vectorStores.retrieve(vectorStoreId);
         setStore(response as VectorStore);
       } catch (err) {
-        setErrorStore(err instanceof Error ? err : new Error("Failed to load vector store."));
+        setErrorStore(
+          err instanceof Error ? err : new Error("Failed to load vector store.")
+        );
       } finally {
         setIsLoadingStore(false);
       }
@@ -61,10 +69,15 @@ export default function FileDetailPage() {
       setIsLoadingFile(true);
       setErrorFile(null);
       try {
-        const response = await client.vectorStores.files.retrieve(vectorStoreId, fileId);
+        const response = await client.vectorStores.files.retrieve(
+          vectorStoreId,
+          fileId
+        );
         setFile(response as VectorStoreFile);
       } catch (err) {
-        setErrorFile(err instanceof Error ? err : new Error("Failed to load file."));
+        setErrorFile(
+          err instanceof Error ? err : new Error("Failed to load file.")
+        );
       } finally {
         setIsLoadingFile(false);
       }
@@ -79,10 +92,15 @@ export default function FileDetailPage() {
       setIsLoadingContents(true);
       setErrorContents(null);
       try {
-        const response = await client.vectorStores.files.content(vectorStoreId, fileId);
+        const response = await client.vectorStores.files.content(
+          vectorStoreId,
+          fileId
+        );
         setContents(response);
       } catch (err) {
-        setErrorContents(err instanceof Error ? err : new Error("Failed to load contents."));
+        setErrorContents(
+          err instanceof Error ? err : new Error("Failed to load contents.")
+        );
       } finally {
         setIsLoadingContents(false);
       }
@@ -91,20 +109,27 @@ export default function FileDetailPage() {
   }, [vectorStoreId, fileId, client]);
 
   const handleViewContents = () => {
-    router.push(`/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents`);
+    router.push(
+      `/logs/vector-stores/${vectorStoreId}/files/${fileId}/contents`
+    );
   };
 
   const title = `File: ${fileId}`;
 
   const breadcrumbSegments: BreadcrumbSegment[] = [
     { label: "Vector Stores", href: "/logs/vector-stores" },
-    { label: store?.name || vectorStoreId, href: `/logs/vector-stores/${vectorStoreId}` },
+    {
+      label: store?.name || vectorStoreId,
+      href: `/logs/vector-stores/${vectorStoreId}`,
+    },
     { label: "Files", href: `/logs/vector-stores/${vectorStoreId}` },
     { label: fileId },
   ];
 
   if (errorStore) {
-    return <DetailErrorView title={title} id={vectorStoreId} error={errorStore} />;
+    return (
+      <DetailErrorView title={title} id={vectorStoreId} error={errorStore} />
+    );
   }
   if (isLoadingStore) {
     return <DetailLoadingView title={title} />;
@@ -136,19 +161,29 @@ export default function FileDetailPage() {
                 <h3 className="text-lg font-medium mb-2">File Details</h3>
                 <div className="grid grid-cols-2 gap-4 text-sm">
                   <div>
-                    <span className="font-medium text-gray-600 dark:text-gray-400">Status:</span>
+                    <span className="font-medium text-gray-600 dark:text-gray-400">
+                      Status:
+                    </span>
                     <span className="ml-2">{file.status}</span>
                   </div>
                   <div>
-                    <span className="font-medium text-gray-600 dark:text-gray-400">Size:</span>
+                    <span className="font-medium text-gray-600 dark:text-gray-400">
+                      Size:
+                    </span>
                     <span className="ml-2">{file.usage_bytes} bytes</span>
                   </div>
                   <div>
-                    <span className="font-medium text-gray-600 dark:text-gray-400">Created:</span>
-                    <span className="ml-2">{new Date(file.created_at * 1000).toLocaleString()}</span>
+                    <span className="font-medium text-gray-600 dark:text-gray-400">
+                      Created:
+                    </span>
+                    <span className="ml-2">
+                      {new Date(file.created_at * 1000).toLocaleString()}
+                    </span>
                   </div>
                   <div>
-                    <span className="font-medium text-gray-600 dark:text-gray-400">Content Strategy:</span>
+                    <span className="font-medium text-gray-600 dark:text-gray-400">
+                      Content Strategy:
+                    </span>
                     <span className="ml-2">{file.chunking_strategy.type}</span>
                   </div>
                 </div>
@@ -166,9 +201,7 @@ export default function FileDetailPage() {
               </div>
             </div>
           ) : (
-            <p className="text-gray-500 italic text-sm">
-              File not found.
-            </p>
+            <p className="text-gray-500 italic text-sm">File not found.</p>
           )}
         </CardContent>
       </Card>
@@ -192,16 +225,27 @@ export default function FileDetailPage() {
             <div className="space-y-3">
               <div className="grid grid-cols-2 gap-4 text-sm">
                 <div>
-                  <span className="font-medium text-gray-600 dark:text-gray-400">Content Items:</span>
+                  <span className="font-medium text-gray-600 dark:text-gray-400">
+                    Content Items:
+                  </span>
                   <span className="ml-2">{contents.content.length}</span>
                 </div>
                 <div>
-                  <span className="font-medium text-gray-600 dark:text-gray-400">Total Characters:</span>
-                  <span className="ml-2">{contents.content.reduce((total, item) => total + item.text.length, 0)}</span>
+                  <span className="font-medium text-gray-600 dark:text-gray-400">
+                    Total Characters:
+                  </span>
+                  <span className="ml-2">
+                    {contents.content.reduce(
+                      (total, item) => total + item.text.length,
+                      0
+                    )}
+                  </span>
                 </div>
               </div>
               <div className="pt-2">
-                <span className="text-sm font-medium text-gray-600 dark:text-gray-400">Preview:</span>
+                <span className="text-sm font-medium text-gray-600 dark:text-gray-400">
+                  Preview:
+                </span>
                 <div className="mt-1 bg-gray-50 dark:bg-gray-800 rounded-md p-3">
                   <p className="text-sm text-gray-900 dark:text-gray-100 line-clamp-3">
                     {contents.content[0]?.text.substring(0, 200)}...
diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/page.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/page.tsx
index f27c9d802..cad50506c 100644
--- a/llama_stack/ui/app/logs/vector-stores/[id]/page.tsx
+++ b/llama_stack/ui/app/logs/vector-stores/[id]/page.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import { useEffect, useState } from "react";
-import { useParams, useRouter } from "next/navigation";
+import { useParams } from "next/navigation";
 import { useAuthClient } from "@/hooks/use-auth-client";
 import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores";
 import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files";
@@ -11,7 +11,6 @@ export default function VectorStoreDetailPage() {
   const params = useParams();
   const id = params.id as string;
   const client = useAuthClient();
-  const router = useRouter();
 
   const [store, setStore] = useState<VectorStore | null>(null);
   const [files, setFiles] = useState<VectorStoreFile[]>([]);
@@ -34,9 +33,7 @@ export default function VectorStoreDetailPage() {
         setStore(response as VectorStore);
       } catch (err) {
         setErrorStore(
-          err instanceof Error
-            ? err
-            : new Error("Failed to load vector store."),
+          err instanceof Error ? err : new Error("Failed to load vector store.")
         );
       } finally {
         setIsLoadingStore(false);
@@ -55,18 +52,18 @@ export default function VectorStoreDetailPage() {
       setIsLoadingFiles(true);
       setErrorFiles(null);
       try {
-        const result = await client.vectorStores.files.list(id as any);
-        setFiles((result as any).data);
+        const result = await client.vectorStores.files.list(id);
+        setFiles((result as { data: VectorStoreFile[] }).data);
       } catch (err) {
         setErrorFiles(
-          err instanceof Error ? err : new Error("Failed to load files."),
+          err instanceof Error ? err : new Error("Failed to load files.")
         );
       } finally {
         setIsLoadingFiles(false);
       }
     };
     fetchFiles();
-  }, [id]);
+  }, [id, client.vectorStores.files]);
 
   return (
     <VectorStoreDetailView
diff --git a/llama_stack/ui/app/logs/vector-stores/page.tsx b/llama_stack/ui/app/logs/vector-stores/page.tsx
index 664513caa..72196d496 100644
--- a/llama_stack/ui/app/logs/vector-stores/page.tsx
+++ b/llama_stack/ui/app/logs/vector-stores/page.tsx
@@ -1,7 +1,6 @@
 "use client";
 
 import React from "react";
-import { useAuthClient } from "@/hooks/use-auth-client";
 import type {
   ListVectorStoresResponse,
   VectorStore,
@@ -12,7 +11,6 @@ import { Button } from "@/components/ui/button";
 import {
   Table,
   TableBody,
-  TableCaption,
   TableCell,
   TableHead,
   TableHeader,
@@ -21,7 +19,6 @@ import {
 import { Skeleton } from "@/components/ui/skeleton";
 
 export default function VectorStoresPage() {
-  const client = useAuthClient();
   const router = useRouter();
   const {
     data: stores,
@@ -37,7 +34,7 @@ export default function VectorStoresPage() {
         after: params.after,
         limit: params.limit,
         order: params.order,
-      } as any);
+      } as Parameters<typeof client.vectorStores.list>[0]);
       return response as ListVectorStoresResponse;
     },
     errorMessagePrefix: "vector stores",
@@ -53,11 +50,11 @@ export default function VectorStoresPage() {
   const renderContent = () => {
     if (status === "loading") {
       return (
-          <div className="space-y-2">
-            <Skeleton className="h-8 w-full"/>
-            <Skeleton className="h-4 w-full"/>
-            <Skeleton className="h-4 w-full"/>
-          </div>
+        <div className="space-y-2">
+          <Skeleton className="h-8 w-full" />
+          <Skeleton className="h-4 w-full" />
+          <Skeleton className="h-4 w-full" />
+        </div>
       );
     }
 
@@ -70,72 +67,72 @@ export default function VectorStoresPage() {
     }
 
     return (
-        <div className="overflow-auto flex-1 min-h-0">
-          <Table>
-            <TableHeader>
-              <TableRow>
-                <TableHead>ID</TableHead>
-                <TableHead>Name</TableHead>
-                <TableHead>Created</TableHead>
-                <TableHead>Completed</TableHead>
-                <TableHead>Cancelled</TableHead>
-                <TableHead>Failed</TableHead>
-                <TableHead>In Progress</TableHead>
-                <TableHead>Total</TableHead>
-                <TableHead>Usage Bytes</TableHead>
-                <TableHead>Provider ID</TableHead>
-                <TableHead>Provider Vector DB ID</TableHead>
-              </TableRow>
-            </TableHeader>
-            <TableBody>
-              {stores.map((store) => {
-                const fileCounts = store.file_counts;
-                const metadata = store.metadata || {};
-                const providerId = metadata.provider_id ?? "";
-                const providerDbId = metadata.provider_vector_db_id ?? "";
+      <div className="overflow-auto flex-1 min-h-0">
+        <Table>
+          <TableHeader>
+            <TableRow>
+              <TableHead>ID</TableHead>
+              <TableHead>Name</TableHead>
+              <TableHead>Created</TableHead>
+              <TableHead>Completed</TableHead>
+              <TableHead>Cancelled</TableHead>
+              <TableHead>Failed</TableHead>
+              <TableHead>In Progress</TableHead>
+              <TableHead>Total</TableHead>
+              <TableHead>Usage Bytes</TableHead>
+              <TableHead>Provider ID</TableHead>
+              <TableHead>Provider Vector DB ID</TableHead>
+            </TableRow>
+          </TableHeader>
+          <TableBody>
+            {stores.map(store => {
+              const fileCounts = store.file_counts;
+              const metadata = store.metadata || {};
+              const providerId = metadata.provider_id ?? "";
+              const providerDbId = metadata.provider_vector_db_id ?? "";
 
-                return (
-                    <TableRow
-                        key={store.id}
-                        onClick={() => router.push(`/logs/vector-stores/${store.id}`)}
-                        className="cursor-pointer hover:bg-muted/50"
+              return (
+                <TableRow
+                  key={store.id}
+                  onClick={() => router.push(`/logs/vector-stores/${store.id}`)}
+                  className="cursor-pointer hover:bg-muted/50"
+                >
+                  <TableCell>
+                    <Button
+                      variant="link"
+                      className="p-0 h-auto font-mono text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300"
+                      onClick={() =>
+                        router.push(`/logs/vector-stores/${store.id}`)
+                      }
                     >
-                    <TableCell>
-                      <Button
-                        variant="link"
-                        className="p-0 h-auto font-mono text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300"
-                        onClick={() =>
-                          router.push(`/logs/vector-stores/${store.id}`)
-                        }
-                      >
-                        {store.id}
-                      </Button>
-                    </TableCell>
-                      <TableCell>{store.name}</TableCell>
-                      <TableCell>
-                        {new Date(store.created_at * 1000).toLocaleString()}
-                      </TableCell>
-                      <TableCell>{fileCounts.completed}</TableCell>
-                      <TableCell>{fileCounts.cancelled}</TableCell>
-                      <TableCell>{fileCounts.failed}</TableCell>
-                      <TableCell>{fileCounts.in_progress}</TableCell>
-                      <TableCell>{fileCounts.total}</TableCell>
-                      <TableCell>{store.usage_bytes}</TableCell>
-                      <TableCell>{providerId}</TableCell>
-                      <TableCell>{providerDbId}</TableCell>
-                    </TableRow>
-                );
-              })}
-            </TableBody>
-          </Table>
-        </div>
+                      {store.id}
+                    </Button>
+                  </TableCell>
+                  <TableCell>{store.name}</TableCell>
+                  <TableCell>
+                    {new Date(store.created_at * 1000).toLocaleString()}
+                  </TableCell>
+                  <TableCell>{fileCounts.completed}</TableCell>
+                  <TableCell>{fileCounts.cancelled}</TableCell>
+                  <TableCell>{fileCounts.failed}</TableCell>
+                  <TableCell>{fileCounts.in_progress}</TableCell>
+                  <TableCell>{fileCounts.total}</TableCell>
+                  <TableCell>{store.usage_bytes}</TableCell>
+                  <TableCell>{providerId}</TableCell>
+                  <TableCell>{providerDbId}</TableCell>
+                </TableRow>
+              );
+            })}
+          </TableBody>
+        </Table>
+      </div>
     );
   };
 
   return (
-      <div className="space-y-4">
-        <h1 className="text-2xl font-semibold">Vector Stores</h1>
-        {renderContent()}
-      </div>
+    <div className="space-y-4">
+      <h1 className="text-2xl font-semibold">Vector Stores</h1>
+      {renderContent()}
+    </div>
   );
 }
diff --git a/llama_stack/ui/components/chat-completions/chat-completion-detail.test.tsx b/llama_stack/ui/components/chat-completions/chat-completion-detail.test.tsx
index 5348dbc3a..52258eda9 100644
--- a/llama_stack/ui/components/chat-completions/chat-completion-detail.test.tsx
+++ b/llama_stack/ui/components/chat-completions/chat-completion-detail.test.tsx
@@ -14,7 +14,7 @@ describe("ChatCompletionDetailView", () => {
         isLoading={true}
         error={null}
         id="test-id"
-      />,
+      />
     );
     // Use the data-slot attribute for Skeletons
     const skeletons = container.querySelectorAll('[data-slot="skeleton"]');
@@ -28,10 +28,10 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={{ name: "Error", message: "Network Error" }}
         id="err-id"
-      />,
+      />
     );
     expect(
-      screen.getByText(/Error loading details for ID err-id: Network Error/),
+      screen.getByText(/Error loading details for ID err-id: Network Error/)
     ).toBeInTheDocument();
   });
 
@@ -42,11 +42,11 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={{ name: "Error", message: "" }}
         id="err-id"
-      />,
+      />
     );
     // Use regex to match the error message regardless of whitespace
     expect(
-      screen.getByText(/Error loading details for ID\s*err-id\s*:/),
+      screen.getByText(/Error loading details for ID\s*err-id\s*:/)
     ).toBeInTheDocument();
   });
 
@@ -57,11 +57,11 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={{} as Error}
         id="err-id"
-      />,
+      />
     );
     // Use regex to match the error message regardless of whitespace
     expect(
-      screen.getByText(/Error loading details for ID\s*err-id\s*:/),
+      screen.getByText(/Error loading details for ID\s*err-id\s*:/)
     ).toBeInTheDocument();
   });
 
@@ -72,10 +72,10 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={null}
         id="notfound-id"
-      />,
+      />
     );
     expect(
-      screen.getByText("No details found for ID: notfound-id."),
+      screen.getByText("No details found for ID: notfound-id.")
     ).toBeInTheDocument();
   });
 
@@ -100,7 +100,7 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={null}
         id={mockCompletion.id}
-      />,
+      />
     );
     // Input
     expect(screen.getByText("Input")).toBeInTheDocument();
@@ -112,7 +112,7 @@ describe("ChatCompletionDetailView", () => {
     expect(screen.getByText("Properties")).toBeInTheDocument();
     expect(screen.getByText("Created:")).toBeInTheDocument();
     expect(
-      screen.getByText(new Date(1710000000 * 1000).toLocaleString()),
+      screen.getByText(new Date(1710000000 * 1000).toLocaleString())
     ).toBeInTheDocument();
     expect(screen.getByText("ID:")).toBeInTheDocument();
     expect(screen.getByText("comp_123")).toBeInTheDocument();
@@ -150,7 +150,7 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={null}
         id={mockCompletion.id}
-      />,
+      />
     );
     // Output should include the tool call block (should be present twice: input and output)
     const toolCallLabels = screen.getAllByText("Tool Call");
@@ -178,13 +178,13 @@ describe("ChatCompletionDetailView", () => {
         isLoading={false}
         error={null}
         id={mockCompletion.id}
-      />,
+      />
     );
     // Input section should be present but empty
     expect(screen.getByText("Input")).toBeInTheDocument();
     // Output section should show fallback message
     expect(
-      screen.getByText("No message found in assistant's choice."),
+      screen.getByText("No message found in assistant's choice.")
     ).toBeInTheDocument();
     // Properties should show N/A for finish reason
     expect(screen.getByText("Finish Reason:")).toBeInTheDocument();
diff --git a/llama_stack/ui/components/chat-completions/chat-completion-detail.tsx b/llama_stack/ui/components/chat-completions/chat-completion-detail.tsx
index 200807864..0d11d2444 100644
--- a/llama_stack/ui/components/chat-completions/chat-completion-detail.tsx
+++ b/llama_stack/ui/components/chat-completions/chat-completion-detail.tsx
@@ -53,14 +53,14 @@ export function ChatCompletionDetailView({
           {completion.choices?.[0]?.message?.tool_calls &&
           Array.isArray(completion.choices[0].message.tool_calls) &&
           !completion.input_messages?.some(
-            (im) =>
+            im =>
               im.role === "assistant" &&
               im.tool_calls &&
               Array.isArray(im.tool_calls) &&
-              im.tool_calls.length > 0,
+              im.tool_calls.length > 0
           )
             ? completion.choices[0].message.tool_calls.map(
-                (toolCall: any, index: number) => {
+                (toolCall: { function?: { name?: string } }, index: number) => {
                   const assistantToolCallMessage: ChatMessage = {
                     role: "assistant",
                     tool_calls: [toolCall],
@@ -72,7 +72,7 @@ export function ChatCompletionDetailView({
                       message={assistantToolCallMessage}
                     />
                   );
-                },
+                }
               )
             : null}
         </CardContent>
@@ -89,7 +89,7 @@ export function ChatCompletionDetailView({
             />
           ) : (
             <p className="text-gray-500 italic text-sm">
-              No message found in assistant's choice.
+              No message found in assistant&apos;s choice.
             </p>
           )}
         </CardContent>
@@ -120,13 +120,18 @@ export function ChatCompletionDetailView({
               value={
                 <div>
                   <ul className="list-disc list-inside pl-4 mt-1">
-                    {toolCalls.map((toolCall: any, index: number) => (
-                      <li key={index}>
-                        <span className="text-gray-900 font-medium">
-                          {toolCall.function?.name || "N/A"}
-                        </span>
-                      </li>
-                    ))}
+                    {toolCalls.map(
+                      (
+                        toolCall: { function?: { name?: string } },
+                        index: number
+                      ) => (
+                        <li key={index}>
+                          <span className="text-gray-900 font-medium">
+                            {toolCall.function?.name || "N/A"}
+                          </span>
+                        </li>
+                      )
+                    )}
                   </ul>
                 </div>
               }
diff --git a/llama_stack/ui/components/chat-completions/chat-completion-table.test.tsx b/llama_stack/ui/components/chat-completions/chat-completion-table.test.tsx
index 9171e0106..1cae95ddf 100644
--- a/llama_stack/ui/components/chat-completions/chat-completion-table.test.tsx
+++ b/llama_stack/ui/components/chat-completions/chat-completion-table.test.tsx
@@ -83,7 +83,7 @@ describe("ChatCompletionsTable", () => {
     // Default pass-through implementations
     truncateText.mockImplementation((text: string | undefined) => text);
     extractTextFromContentPart.mockImplementation((content: unknown) =>
-      typeof content === "string" ? content : "extracted text",
+      typeof content === "string" ? content : "extracted text"
     );
     extractDisplayableText.mockImplementation((message: unknown) => {
       const msg = message as { content?: string };
@@ -138,7 +138,7 @@ describe("ChatCompletionsTable", () => {
     if (row) {
       fireEvent.click(row);
       expect(mockPush).toHaveBeenCalledWith(
-        "/logs/chat-completions/completion_123",
+        "/logs/chat-completions/completion_123"
       );
     } else {
       throw new Error('Row with "Test prompt" not found for router mock test.');
@@ -162,7 +162,7 @@ describe("ChatCompletionsTable", () => {
       expect(tableCaption).toBeInTheDocument();
       if (tableCaption) {
         const captionSkeleton = tableCaption.querySelector(
-          '[data-slot="skeleton"]',
+          '[data-slot="skeleton"]'
         );
         expect(captionSkeleton).toBeInTheDocument();
       }
@@ -172,7 +172,7 @@ describe("ChatCompletionsTable", () => {
       expect(tableBody).toBeInTheDocument();
       if (tableBody) {
         const bodySkeletons = tableBody.querySelectorAll(
-          '[data-slot="skeleton"]',
+          '[data-slot="skeleton"]'
         );
         expect(bodySkeletons.length).toBeGreaterThan(0);
       }
@@ -192,14 +192,14 @@ describe("ChatCompletionsTable", () => {
 
       render(<ChatCompletionsTable {...defaultProps} />);
       expect(
-        screen.getByText("Unable to load chat completions"),
+        screen.getByText("Unable to load chat completions")
       ).toBeInTheDocument();
       expect(screen.getByText(errorMessage)).toBeInTheDocument();
     });
 
     test.each([{ name: "Error", message: "" }, {}])(
       "renders default error message when error has no message",
-      (errorObject) => {
+      errorObject => {
         mockedUsePagination.mockReturnValue({
           data: [],
           status: "error",
@@ -210,14 +210,14 @@ describe("ChatCompletionsTable", () => {
 
         render(<ChatCompletionsTable {...defaultProps} />);
         expect(
-          screen.getByText("Unable to load chat completions"),
+          screen.getByText("Unable to load chat completions")
         ).toBeInTheDocument();
         expect(
           screen.getByText(
-            "An unexpected error occurred while loading the data.",
-          ),
+            "An unexpected error occurred while loading the data."
+          )
         ).toBeInTheDocument();
-      },
+      }
     );
   });
 
@@ -225,7 +225,7 @@ describe("ChatCompletionsTable", () => {
     test('renders "No chat completions found." and no table when data array is empty', () => {
       render(<ChatCompletionsTable {...defaultProps} />);
       expect(
-        screen.getByText("No chat completions found."),
+        screen.getByText("No chat completions found.")
       ).toBeInTheDocument();
 
       // Ensure that the table structure is NOT rendered in the empty state
@@ -292,7 +292,7 @@ describe("ChatCompletionsTable", () => {
 
       // Table caption
       expect(
-        screen.getByText("A list of your recent chat completions."),
+        screen.getByText("A list of your recent chat completions.")
       ).toBeInTheDocument();
 
       // Table headers
@@ -306,14 +306,14 @@ describe("ChatCompletionsTable", () => {
       expect(screen.getByText("Test output")).toBeInTheDocument();
       expect(screen.getByText("llama-test-model")).toBeInTheDocument();
       expect(
-        screen.getByText(new Date(1710000000 * 1000).toLocaleString()),
+        screen.getByText(new Date(1710000000 * 1000).toLocaleString())
       ).toBeInTheDocument();
 
       expect(screen.getByText("Another input")).toBeInTheDocument();
       expect(screen.getByText("Another output")).toBeInTheDocument();
       expect(screen.getByText("llama-another-model")).toBeInTheDocument();
       expect(
-        screen.getByText(new Date(1710001000 * 1000).toLocaleString()),
+        screen.getByText(new Date(1710001000 * 1000).toLocaleString())
       ).toBeInTheDocument();
     });
   });
@@ -328,7 +328,7 @@ describe("ChatCompletionsTable", () => {
           return typeof text === "string" && text.length > effectiveMaxLength
             ? text.slice(0, effectiveMaxLength) + "..."
             : text;
-        },
+        }
       );
 
       const longInput =
@@ -368,7 +368,7 @@ describe("ChatCompletionsTable", () => {
 
       // The truncated text should be present for both input and output
       const truncatedTexts = screen.getAllByText(
-        longInput.slice(0, 10) + "...",
+        longInput.slice(0, 10) + "..."
       );
       expect(truncatedTexts.length).toBe(2); // one for input, one for output
     });
@@ -420,7 +420,7 @@ describe("ChatCompletionsTable", () => {
       // Verify the extracted text appears in the table
       expect(screen.getByText("Extracted input")).toBeInTheDocument();
       expect(
-        screen.getByText("Extracted output from assistant"),
+        screen.getByText("Extracted output from assistant")
       ).toBeInTheDocument();
     });
   });
diff --git a/llama_stack/ui/components/chat-completions/chat-completions-table.tsx b/llama_stack/ui/components/chat-completions/chat-completions-table.tsx
index 65f6c71af..64e8167f2 100644
--- a/llama_stack/ui/components/chat-completions/chat-completions-table.tsx
+++ b/llama_stack/ui/components/chat-completions/chat-completions-table.tsx
@@ -5,6 +5,7 @@ import {
   UsePaginationOptions,
   ListChatCompletionsResponse,
 } from "@/lib/types";
+import { ListChatCompletionsParams } from "@/lib/llama-stack-client";
 import { LogsTable, LogTableRow } from "@/components/logs/logs-table";
 import {
   extractTextFromContentPart,
@@ -38,14 +39,14 @@ export function ChatCompletionsTable({
       limit: number;
       model?: string;
       order?: string;
-    },
+    }
   ) => {
     const response = await client.chat.completions.list({
       after: params.after,
       limit: params.limit,
       ...(params.model && { model: params.model }),
       ...(params.order && { order: params.order }),
-    } as any);
+    } as ListChatCompletionsParams);
 
     return response as ListChatCompletionsResponse;
   };
diff --git a/llama_stack/ui/components/chat-completions/chat-messasge-item.tsx b/llama_stack/ui/components/chat-completions/chat-messasge-item.tsx
index 6170e816e..de097e630 100644
--- a/llama_stack/ui/components/chat-completions/chat-messasge-item.tsx
+++ b/llama_stack/ui/components/chat-completions/chat-messasge-item.tsx
@@ -37,21 +37,26 @@ export function ChatMessageItem({ message }: ChatMessageItemProps) {
       ) {
         return (
           <>
-            {message.tool_calls.map((toolCall: any, index: number) => {
-              const formattedToolCall = formatToolCallToString(toolCall);
-              const toolCallContent = (
-                <ToolCallBlock>
-                  {formattedToolCall || "Error: Could not display tool call"}
-                </ToolCallBlock>
-              );
-              return (
-                <MessageBlock
-                  key={index}
-                  label="Tool Call"
-                  content={toolCallContent}
-                />
-              );
-            })}
+            {message.tool_calls.map(
+              (
+                toolCall: { function?: { name?: string; arguments?: unknown } },
+                index: number
+              ) => {
+                const formattedToolCall = formatToolCallToString(toolCall);
+                const toolCallContent = (
+                  <ToolCallBlock>
+                    {formattedToolCall || "Error: Could not display tool call"}
+                  </ToolCallBlock>
+                );
+                return (
+                  <MessageBlock
+                    key={index}
+                    label="Tool Call"
+                    content={toolCallContent}
+                  />
+                );
+              }
+            )}
           </>
         );
       } else {
diff --git a/llama_stack/ui/components/chat-playground/chat-message.tsx b/llama_stack/ui/components/chat-playground/chat-message.tsx
index e5d621c81..84c798e29 100644
--- a/llama_stack/ui/components/chat-playground/chat-message.tsx
+++ b/llama_stack/ui/components/chat-playground/chat-message.tsx
@@ -1,18 +1,18 @@
-"use client"
+"use client";
 
-import React, { useMemo, useState } from "react"
-import { cva, type VariantProps } from "class-variance-authority"
-import { motion } from "framer-motion"
-import { Ban, ChevronRight, Code2, Loader2, Terminal } from "lucide-react"
+import React, { useMemo, useState } from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+import { motion } from "framer-motion";
+import { Ban, ChevronRight, Code2, Loader2, Terminal } from "lucide-react";
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils";
 import {
   Collapsible,
   CollapsibleContent,
   CollapsibleTrigger,
-} from "@/components/ui/collapsible"
-import { FilePreview } from "@/components/ui/file-preview"
-import { MarkdownRenderer } from "@/components/chat-playground/markdown-renderer"
+} from "@/components/ui/collapsible";
+import { FilePreview } from "@/components/ui/file-preview";
+import { MarkdownRenderer } from "@/components/chat-playground/markdown-renderer";
 
 const chatBubbleVariants = cva(
   "group/message relative break-words rounded-lg p-3 text-sm sm:max-w-[70%]",
@@ -52,66 +52,66 @@ const chatBubbleVariants = cva(
       },
     ],
   }
-)
+);
 
-type Animation = VariantProps<typeof chatBubbleVariants>["animation"]
+type Animation = VariantProps<typeof chatBubbleVariants>["animation"];
 
 interface Attachment {
-  name?: string
-  contentType?: string
-  url: string
+  name?: string;
+  contentType?: string;
+  url: string;
 }
 
 interface PartialToolCall {
-  state: "partial-call"
-  toolName: string
+  state: "partial-call";
+  toolName: string;
 }
 
 interface ToolCall {
-  state: "call"
-  toolName: string
+  state: "call";
+  toolName: string;
 }
 
 interface ToolResult {
-  state: "result"
-  toolName: string
+  state: "result";
+  toolName: string;
   result: {
-    __cancelled?: boolean
-    [key: string]: any
-  }
+    __cancelled?: boolean;
+    [key: string]: unknown;
+  };
 }
 
-type ToolInvocation = PartialToolCall | ToolCall | ToolResult
+type ToolInvocation = PartialToolCall | ToolCall | ToolResult;
 
 interface ReasoningPart {
-  type: "reasoning"
-  reasoning: string
+  type: "reasoning";
+  reasoning: string;
 }
 
 interface ToolInvocationPart {
-  type: "tool-invocation"
-  toolInvocation: ToolInvocation
+  type: "tool-invocation";
+  toolInvocation: ToolInvocation;
 }
 
 interface TextPart {
-  type: "text"
-  text: string
+  type: "text";
+  text: string;
 }
 
 // For compatibility with AI SDK types, not used
 interface SourcePart {
-  type: "source"
-  source?: any
+  type: "source";
+  source?: unknown;
 }
 
 interface FilePart {
-  type: "file"
-  mimeType: string
-  data: string
+  type: "file";
+  mimeType: string;
+  data: string;
 }
 
 interface StepStartPart {
-  type: "step-start"
+  type: "step-start";
 }
 
 type MessagePart =
@@ -120,22 +120,22 @@ type MessagePart =
   | ToolInvocationPart
   | SourcePart
   | FilePart
-  | StepStartPart
+  | StepStartPart;
 
 export interface Message {
-  id: string
-  role: "user" | "assistant" | (string & {})
-  content: string
-  createdAt?: Date
-  experimental_attachments?: Attachment[]
-  toolInvocations?: ToolInvocation[]
-  parts?: MessagePart[]
+  id: string;
+  role: "user" | "assistant" | (string & {});
+  content: string;
+  createdAt?: Date;
+  experimental_attachments?: Attachment[];
+  toolInvocations?: ToolInvocation[];
+  parts?: MessagePart[];
 }
 
 export interface ChatMessageProps extends Message {
-  showTimeStamp?: boolean
-  animation?: Animation
-  actions?: React.ReactNode
+  showTimeStamp?: boolean;
+  animation?: Animation;
+  actions?: React.ReactNode;
 }
 
 export const ChatMessage: React.FC<ChatMessageProps> = ({
@@ -150,21 +150,21 @@ export const ChatMessage: React.FC<ChatMessageProps> = ({
   parts,
 }) => {
   const files = useMemo(() => {
-    return experimental_attachments?.map((attachment) => {
-      const dataArray = dataUrlToUint8Array(attachment.url)
+    return experimental_attachments?.map(attachment => {
+      const dataArray = dataUrlToUint8Array(attachment.url);
       const file = new File([dataArray], attachment.name ?? "Unknown", {
         type: attachment.contentType,
-      })
-      return file
-    })
-  }, [experimental_attachments])
+      });
+      return file;
+    });
+  }, [experimental_attachments]);
 
-  const isUser = role === "user"
+  const isUser = role === "user";
 
   const formattedTime = createdAt?.toLocaleTimeString("en-US", {
     hour: "2-digit",
     minute: "2-digit",
-  })
+  });
 
   if (isUser) {
     return (
@@ -174,7 +174,7 @@ export const ChatMessage: React.FC<ChatMessageProps> = ({
         {files ? (
           <div className="mb-1 flex flex-wrap gap-2">
             {files.map((file, index) => {
-              return <FilePreview file={file} key={index} />
+              return <FilePreview file={file} key={index} />;
             })}
           </div>
         ) : null}
@@ -195,7 +195,7 @@ export const ChatMessage: React.FC<ChatMessageProps> = ({
           </time>
         ) : null}
       </div>
-    )
+    );
   }
 
   if (parts && parts.length > 0) {
@@ -230,23 +230,23 @@ export const ChatMessage: React.FC<ChatMessageProps> = ({
               </time>
             ) : null}
           </div>
-        )
+        );
       } else if (part.type === "reasoning") {
-        return <ReasoningBlock key={`reasoning-${index}`} part={part} />
+        return <ReasoningBlock key={`reasoning-${index}`} part={part} />;
       } else if (part.type === "tool-invocation") {
         return (
           <ToolCall
             key={`tool-${index}`}
             toolInvocations={[part.toolInvocation]}
           />
-        )
+        );
       }
-      return null
-    })
+      return null;
+    });
   }
 
   if (toolInvocations && toolInvocations.length > 0) {
-    return <ToolCall toolInvocations={toolInvocations} />
+    return <ToolCall toolInvocations={toolInvocations} />;
   }
 
   return (
@@ -272,17 +272,17 @@ export const ChatMessage: React.FC<ChatMessageProps> = ({
         </time>
       ) : null}
     </div>
-  )
-}
+  );
+};
 
 function dataUrlToUint8Array(data: string) {
-  const base64 = data.split(",")[1]
-  const buf = Buffer.from(base64, "base64")
-  return new Uint8Array(buf)
+  const base64 = data.split(",")[1];
+  const buf = Buffer.from(base64, "base64");
+  return new Uint8Array(buf);
 }
 
 const ReasoningBlock = ({ part }: { part: ReasoningPart }) => {
-  const [isOpen, setIsOpen] = useState(false)
+  const [isOpen, setIsOpen] = useState(false);
 
   return (
     <div className="mb-2 flex flex-col items-start sm:max-w-[70%]">
@@ -319,20 +319,20 @@ const ReasoningBlock = ({ part }: { part: ReasoningPart }) => {
         </CollapsibleContent>
       </Collapsible>
     </div>
-  )
-}
+  );
+};
 
 function ToolCall({
   toolInvocations,
 }: Pick<ChatMessageProps, "toolInvocations">) {
-  if (!toolInvocations?.length) return null
+  if (!toolInvocations?.length) return null;
 
   return (
     <div className="flex flex-col items-start gap-2">
       {toolInvocations.map((invocation, index) => {
         const isCancelled =
           invocation.state === "result" &&
-          invocation.result.__cancelled === true
+          invocation.result.__cancelled === true;
 
         if (isCancelled) {
           return (
@@ -350,7 +350,7 @@ function ToolCall({
                 </span>
               </span>
             </div>
-          )
+          );
         }
 
         switch (invocation.state) {
@@ -373,7 +373,7 @@ function ToolCall({
                 </span>
                 <Loader2 className="h-3 w-3 animate-spin" />
               </div>
-            )
+            );
           case "result":
             return (
               <div
@@ -395,11 +395,11 @@ function ToolCall({
                   {JSON.stringify(invocation.result, null, 2)}
                 </pre>
               </div>
-            )
+            );
           default:
-            return null
+            return null;
         }
       })}
     </div>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/chat-playground/chat.tsx b/llama_stack/ui/components/chat-playground/chat.tsx
index ee83fd9bb..023bf0728 100644
--- a/llama_stack/ui/components/chat-playground/chat.tsx
+++ b/llama_stack/ui/components/chat-playground/chat.tsx
@@ -1,4 +1,4 @@
-"use client"
+"use client";
 
 import {
   forwardRef,
@@ -6,48 +6,48 @@ import {
   useRef,
   useState,
   type ReactElement,
-} from "react"
-import { ArrowDown, ThumbsDown, ThumbsUp } from "lucide-react"
+} from "react";
+import { ArrowDown, ThumbsDown, ThumbsUp } from "lucide-react";
 
-import { cn } from "@/lib/utils"
-import { useAutoScroll } from "@/hooks/use-auto-scroll"
-import { Button } from "@/components/ui/button"
-import { type Message } from "@/components/chat-playground/chat-message"
-import { CopyButton } from "@/components/ui/copy-button"
-import { MessageInput } from "@/components/chat-playground/message-input"
-import { MessageList } from "@/components/chat-playground/message-list"
-import { PromptSuggestions } from "@/components/chat-playground/prompt-suggestions"
+import { cn } from "@/lib/utils";
+import { useAutoScroll } from "@/hooks/use-auto-scroll";
+import { Button } from "@/components/ui/button";
+import { type Message } from "@/components/chat-playground/chat-message";
+import { CopyButton } from "@/components/ui/copy-button";
+import { MessageInput } from "@/components/chat-playground/message-input";
+import { MessageList } from "@/components/chat-playground/message-list";
+import { PromptSuggestions } from "@/components/chat-playground/prompt-suggestions";
 
 interface ChatPropsBase {
   handleSubmit: (
     event?: { preventDefault?: () => void },
     options?: { experimental_attachments?: FileList }
-  ) => void
-  messages: Array<Message>
-  input: string
-  className?: string
-  handleInputChange: React.ChangeEventHandler<HTMLTextAreaElement>
-  isGenerating: boolean
-  stop?: () => void
+  ) => void;
+  messages: Array<Message>;
+  input: string;
+  className?: string;
+  handleInputChange: React.ChangeEventHandler<HTMLTextAreaElement>;
+  isGenerating: boolean;
+  stop?: () => void;
   onRateResponse?: (
     messageId: string,
     rating: "thumbs-up" | "thumbs-down"
-  ) => void
-  setMessages?: (messages: any[]) => void
-  transcribeAudio?: (blob: Blob) => Promise<string>
+  ) => void;
+  setMessages?: (messages: Message[]) => void;
+  transcribeAudio?: (blob: Blob) => Promise<string>;
 }
 
 interface ChatPropsWithoutSuggestions extends ChatPropsBase {
-  append?: never
-  suggestions?: never
+  append?: never;
+  suggestions?: never;
 }
 
 interface ChatPropsWithSuggestions extends ChatPropsBase {
-  append: (message: { role: "user"; content: string }) => void
-  suggestions: string[]
+  append: (message: { role: "user"; content: string }) => void;
+  suggestions: string[];
 }
 
-type ChatProps = ChatPropsWithoutSuggestions | ChatPropsWithSuggestions
+type ChatProps = ChatPropsWithoutSuggestions | ChatPropsWithSuggestions;
 
 export function Chat({
   messages,
@@ -63,34 +63,34 @@ export function Chat({
   setMessages,
   transcribeAudio,
 }: ChatProps) {
-  const lastMessage = messages.at(-1)
-  const isEmpty = messages.length === 0
-  const isTyping = lastMessage?.role === "user"
+  const lastMessage = messages.at(-1);
+  const isEmpty = messages.length === 0;
+  const isTyping = lastMessage?.role === "user";
 
-  const messagesRef = useRef(messages)
-  messagesRef.current = messages
+  const messagesRef = useRef(messages);
+  messagesRef.current = messages;
 
   // Enhanced stop function that marks pending tool calls as cancelled
   const handleStop = useCallback(() => {
-    stop?.()
+    stop?.();
 
-    if (!setMessages) return
+    if (!setMessages) return;
 
-    const latestMessages = [...messagesRef.current]
+    const latestMessages = [...messagesRef.current];
     const lastAssistantMessage = latestMessages.findLast(
-      (m) => m.role === "assistant"
-    )
+      m => m.role === "assistant"
+    );
 
-    if (!lastAssistantMessage) return
+    if (!lastAssistantMessage) return;
 
-    let needsUpdate = false
-    let updatedMessage = { ...lastAssistantMessage }
+    let needsUpdate = false;
+    let updatedMessage = { ...lastAssistantMessage };
 
     if (lastAssistantMessage.toolInvocations) {
       const updatedToolInvocations = lastAssistantMessage.toolInvocations.map(
-        (toolInvocation) => {
+        toolInvocation => {
           if (toolInvocation.state === "call") {
-            needsUpdate = true
+            needsUpdate = true;
             return {
               ...toolInvocation,
               state: "result",
@@ -98,61 +98,66 @@ export function Chat({
                 content: "Tool execution was cancelled",
                 __cancelled: true, // Special marker to indicate cancellation
               },
-            } as const
+            } as const;
           }
-          return toolInvocation
+          return toolInvocation;
         }
-      )
+      );
 
       if (needsUpdate) {
         updatedMessage = {
           ...updatedMessage,
           toolInvocations: updatedToolInvocations,
-        }
+        };
       }
     }
 
     if (lastAssistantMessage.parts && lastAssistantMessage.parts.length > 0) {
-      const updatedParts = lastAssistantMessage.parts.map((part: any) => {
-        if (
-          part.type === "tool-invocation" &&
-          part.toolInvocation &&
-          part.toolInvocation.state === "call"
-        ) {
-          needsUpdate = true
-          return {
-            ...part,
-            toolInvocation: {
-              ...part.toolInvocation,
-              state: "result",
-              result: {
-                content: "Tool execution was cancelled",
-                __cancelled: true,
+      const updatedParts = lastAssistantMessage.parts.map(
+        (part: {
+          type: string;
+          toolInvocation?: { state: string; toolName: string };
+        }) => {
+          if (
+            part.type === "tool-invocation" &&
+            part.toolInvocation &&
+            part.toolInvocation.state === "call"
+          ) {
+            needsUpdate = true;
+            return {
+              ...part,
+              toolInvocation: {
+                ...part.toolInvocation,
+                state: "result",
+                result: {
+                  content: "Tool execution was cancelled",
+                  __cancelled: true,
+                },
               },
-            },
+            };
           }
+          return part;
         }
-        return part
-      })
+      );
 
       if (needsUpdate) {
         updatedMessage = {
           ...updatedMessage,
           parts: updatedParts,
-        }
+        };
       }
     }
 
     if (needsUpdate) {
       const messageIndex = latestMessages.findIndex(
-        (m) => m.id === lastAssistantMessage.id
-      )
+        m => m.id === lastAssistantMessage.id
+      );
       if (messageIndex !== -1) {
-        latestMessages[messageIndex] = updatedMessage
-        setMessages(latestMessages)
+        latestMessages[messageIndex] = updatedMessage;
+        setMessages(latestMessages);
       }
     }
-  }, [stop, setMessages, messagesRef])
+  }, [stop, setMessages, messagesRef]);
 
   const messageOptions = useCallback(
     (message: Message) => ({
@@ -189,7 +194,7 @@ export function Chat({
       ),
     }),
     [onRateResponse]
-  )
+  );
 
   return (
     <ChatContainer className={className}>
@@ -237,15 +242,15 @@ export function Chat({
         </div>
       </div>
     </ChatContainer>
-  )
+  );
 }
-Chat.displayName = "Chat"
+Chat.displayName = "Chat";
 
 export function ChatMessages({
   messages,
   children,
 }: React.PropsWithChildren<{
-  messages: Message[]
+  messages: Message[];
 }>) {
   const {
     containerRef,
@@ -253,7 +258,7 @@ export function ChatMessages({
     handleScroll,
     shouldAutoScroll,
     handleTouchStart,
-  } = useAutoScroll([messages])
+  } = useAutoScroll([messages]);
 
   return (
     <div
@@ -281,7 +286,7 @@ export function ChatMessages({
         </div>
       )}
     </div>
-  )
+  );
 }
 
 export const ChatContainer = forwardRef<
@@ -294,56 +299,56 @@ export const ChatContainer = forwardRef<
       className={cn("flex flex-col max-h-full w-full", className)}
       {...props}
     />
-  )
-})
-ChatContainer.displayName = "ChatContainer"
+  );
+});
+ChatContainer.displayName = "ChatContainer";
 
 interface ChatFormProps {
-  className?: string
-  isPending: boolean
+  className?: string;
+  isPending: boolean;
   handleSubmit: (
     event?: { preventDefault?: () => void },
     options?: { experimental_attachments?: FileList }
-  ) => void
+  ) => void;
   children: (props: {
-    files: File[] | null
-    setFiles: React.Dispatch<React.SetStateAction<File[] | null>>
-  }) => ReactElement
+    files: File[] | null;
+    setFiles: React.Dispatch<React.SetStateAction<File[] | null>>;
+  }) => ReactElement;
 }
 
 export const ChatForm = forwardRef<HTMLFormElement, ChatFormProps>(
   ({ children, handleSubmit, isPending, className }, ref) => {
-    const [files, setFiles] = useState<File[] | null>(null)
+    const [files, setFiles] = useState<File[] | null>(null);
 
     const onSubmit = (event: React.FormEvent) => {
-      // if (isPending) {
-      //   event.preventDefault()
-      //   return
-      // }
-
-      if (!files) {
-        handleSubmit(event)
-        return
+      if (isPending) {
+        event.preventDefault();
+        return;
       }
 
-      const fileList = createFileList(files)
-      handleSubmit(event, { experimental_attachments: fileList })
-      setFiles(null)
-    }
+      if (!files) {
+        handleSubmit(event);
+        return;
+      }
+
+      const fileList = createFileList(files);
+      handleSubmit(event, { experimental_attachments: fileList });
+      setFiles(null);
+    };
 
     return (
       <form ref={ref} onSubmit={onSubmit} className={className}>
         {children({ files, setFiles })}
       </form>
-    )
+    );
   }
-)
-ChatForm.displayName = "ChatForm"
+);
+ChatForm.displayName = "ChatForm";
 
 function createFileList(files: File[] | FileList): FileList {
-  const dataTransfer = new DataTransfer()
+  const dataTransfer = new DataTransfer();
   for (const file of Array.from(files)) {
-    dataTransfer.items.add(file)
+    dataTransfer.items.add(file);
   }
-  return dataTransfer.files
+  return dataTransfer.files;
 }
diff --git a/llama_stack/ui/components/chat-playground/interrupt-prompt.tsx b/llama_stack/ui/components/chat-playground/interrupt-prompt.tsx
index 757863c62..157de7da1 100644
--- a/llama_stack/ui/components/chat-playground/interrupt-prompt.tsx
+++ b/llama_stack/ui/components/chat-playground/interrupt-prompt.tsx
@@ -1,11 +1,11 @@
-"use client"
+"use client";
 
-import { AnimatePresence, motion } from "framer-motion"
-import { X } from "lucide-react"
+import { AnimatePresence, motion } from "framer-motion";
+import { X } from "lucide-react";
 
 interface InterruptPromptProps {
-  isOpen: boolean
-  close: () => void
+  isOpen: boolean;
+  close: () => void;
 }
 
 export function InterruptPrompt({ isOpen, close }: InterruptPromptProps) {
@@ -37,5 +37,5 @@ export function InterruptPrompt({ isOpen, close }: InterruptPromptProps) {
         </motion.div>
       )}
     </AnimatePresence>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/chat-playground/markdown-renderer.tsx b/llama_stack/ui/components/chat-playground/markdown-renderer.tsx
index 1c2781eaf..bc6bf5122 100644
--- a/llama_stack/ui/components/chat-playground/markdown-renderer.tsx
+++ b/llama_stack/ui/components/chat-playground/markdown-renderer.tsx
@@ -1,12 +1,12 @@
-import React, { Suspense, useEffect, useState } from "react"
-import Markdown from "react-markdown"
-import remarkGfm from "remark-gfm"
+import React, { Suspense, useEffect, useState } from "react";
+import Markdown from "react-markdown";
+import remarkGfm from "remark-gfm";
 
-import { cn } from "@/lib/utils"
-import { CopyButton } from "@/components/ui/copy-button"
+import { cn } from "@/lib/utils";
+import { CopyButton } from "@/components/ui/copy-button";
 
 interface MarkdownRendererProps {
-  children: string
+  children: string;
 }
 
 export function MarkdownRenderer({ children }: MarkdownRendererProps) {
@@ -16,34 +16,34 @@ export function MarkdownRenderer({ children }: MarkdownRendererProps) {
         {children}
       </Markdown>
     </div>
-  )
+  );
 }
 
 interface HighlightedPre extends React.HTMLAttributes<HTMLPreElement> {
-  children: string
-  language: string
+  children: string;
+  language: string;
 }
 
 const HighlightedPre = React.memo(
   ({ children, language, ...props }: HighlightedPre) => {
-    const [tokens, setTokens] = useState<any[] | null>(null)
-    const [isSupported, setIsSupported] = useState(false)
+    const [tokens, setTokens] = useState<unknown[] | null>(null);
+    const [isSupported, setIsSupported] = useState(false);
 
     useEffect(() => {
-      let mounted = true
+      let mounted = true;
 
       const loadAndHighlight = async () => {
         try {
-          const { codeToTokens, bundledLanguages } = await import("shiki")
+          const { codeToTokens, bundledLanguages } = await import("shiki");
 
-          if (!mounted) return
+          if (!mounted) return;
 
           if (!(language in bundledLanguages)) {
-            setIsSupported(false)
-            return
+            setIsSupported(false);
+            return;
           }
 
-          setIsSupported(true)
+          setIsSupported(true);
 
           const { tokens: highlightedTokens } = await codeToTokens(children, {
             lang: language as keyof typeof bundledLanguages,
@@ -52,31 +52,31 @@ const HighlightedPre = React.memo(
               light: "github-light",
               dark: "github-dark",
             },
-          })
+          });
 
           if (mounted) {
-            setTokens(highlightedTokens)
+            setTokens(highlightedTokens);
           }
-        } catch (error) {
+        } catch {
           if (mounted) {
-            setIsSupported(false)
+            setIsSupported(false);
           }
         }
-      }
+      };
 
-      loadAndHighlight()
+      loadAndHighlight();
 
       return () => {
-        mounted = false
-      }
-    }, [children, language])
+        mounted = false;
+      };
+    }, [children, language]);
 
     if (!isSupported) {
-      return <pre {...props}>{children}</pre>
+      return <pre {...props}>{children}</pre>;
     }
 
     if (!tokens) {
-      return <pre {...props}>{children}</pre>
+      return <pre {...props}>{children}</pre>;
     }
 
     return (
@@ -89,7 +89,7 @@ const HighlightedPre = React.memo(
                   const style =
                     typeof token.htmlStyle === "string"
                       ? undefined
-                      : token.htmlStyle
+                      : token.htmlStyle;
 
                   return (
                     <span
@@ -99,7 +99,7 @@ const HighlightedPre = React.memo(
                     >
                       {token.content}
                     </span>
-                  )
+                  );
                 })}
               </span>
               {lineIndex !== tokens.length - 1 && "\n"}
@@ -107,15 +107,15 @@ const HighlightedPre = React.memo(
           ))}
         </code>
       </pre>
-    )
+    );
   }
-)
-HighlightedPre.displayName = "HighlightedCode"
+);
+HighlightedPre.displayName = "HighlightedCode";
 
 interface CodeBlockProps extends React.HTMLAttributes<HTMLPreElement> {
-  children: React.ReactNode
-  className?: string
-  language: string
+  children: React.ReactNode;
+  className?: string;
+  language: string;
 }
 
 const CodeBlock = ({
@@ -127,12 +127,12 @@ const CodeBlock = ({
   const code =
     typeof children === "string"
       ? children
-      : childrenTakeAllStringContents(children)
+      : childrenTakeAllStringContents(children);
 
   const preClass = cn(
     "overflow-x-scroll rounded-md border bg-background/50 p-4 font-mono text-sm [scrollbar-width:none]",
     className
-  )
+  );
 
   return (
     <div className="group/code relative mb-4">
@@ -152,27 +152,27 @@ const CodeBlock = ({
         <CopyButton content={code} copyMessage="Copied code to clipboard" />
       </div>
     </div>
-  )
-}
+  );
+};
 
-function childrenTakeAllStringContents(element: any): string {
+function childrenTakeAllStringContents(element: unknown): string {
   if (typeof element === "string") {
-    return element
+    return element;
   }
 
   if (element?.props?.children) {
-    let children = element.props.children
+    const children = element.props.children;
 
     if (Array.isArray(children)) {
       return children
-        .map((child) => childrenTakeAllStringContents(child))
-        .join("")
+        .map(child => childrenTakeAllStringContents(child))
+        .join("");
     } else {
-      return childrenTakeAllStringContents(children)
+      return childrenTakeAllStringContents(children);
     }
   }
 
-  return ""
+  return "";
 }
 
 const COMPONENTS = {
@@ -184,8 +184,14 @@ const COMPONENTS = {
   strong: withClass("strong", "font-semibold"),
   a: withClass("a", "text-primary underline underline-offset-2"),
   blockquote: withClass("blockquote", "border-l-2 border-primary pl-4"),
-  code: ({ children, className, node, ...rest }: any) => {
-    const match = /language-(\w+)/.exec(className || "")
+  code: ({
+    children,
+    className,
+  }: {
+    children: React.ReactNode;
+    className?: string;
+  }) => {
+    const match = /language-(\w+)/.exec(className || "");
     return match ? (
       <CodeBlock className={className} language={match[1]} {...rest}>
         {children}
@@ -199,9 +205,9 @@ const COMPONENTS = {
       >
         {children}
       </code>
-    )
+    );
   },
-  pre: ({ children }: any) => children,
+  pre: ({ children }: { children: React.ReactNode }) => children,
   ol: withClass("ol", "list-decimal space-y-2 pl-6"),
   ul: withClass("ul", "list-disc space-y-2 pl-6"),
   li: withClass("li", "my-1.5"),
@@ -220,14 +226,14 @@ const COMPONENTS = {
   tr: withClass("tr", "m-0 border-t p-0 even:bg-muted"),
   p: withClass("p", "whitespace-pre-wrap"),
   hr: withClass("hr", "border-foreground/20"),
-}
+};
 
 function withClass(Tag: keyof JSX.IntrinsicElements, classes: string) {
-  const Component = ({ node, ...props }: any) => (
+  const Component = ({ ...props }: Record<string, unknown>) => (
     <Tag className={classes} {...props} />
-  )
-  Component.displayName = Tag
-  return Component
+  );
+  Component.displayName = Tag;
+  return Component;
 }
 
-export default MarkdownRenderer
+export default MarkdownRenderer;
diff --git a/llama_stack/ui/components/chat-playground/message-input.tsx b/llama_stack/ui/components/chat-playground/message-input.tsx
index 4a29386d9..8cfa73b30 100644
--- a/llama_stack/ui/components/chat-playground/message-input.tsx
+++ b/llama_stack/ui/components/chat-playground/message-input.tsx
@@ -1,41 +1,41 @@
-"use client"
+"use client";
 
-import React, { useEffect, useRef, useState } from "react"
-import { AnimatePresence, motion } from "framer-motion"
-import { ArrowUp, Info, Loader2, Mic, Paperclip, Square } from "lucide-react"
-import { omit } from "remeda"
+import React, { useEffect, useRef, useState } from "react";
+import { AnimatePresence, motion } from "framer-motion";
+import { ArrowUp, Info, Loader2, Mic, Paperclip, Square } from "lucide-react";
+import { omit } from "remeda";
 
-import { cn } from "@/lib/utils"
-import { useAudioRecording } from "@/hooks/use-audio-recording"
-import { useAutosizeTextArea } from "@/hooks/use-autosize-textarea"
-import { AudioVisualizer } from "@/components/ui/audio-visualizer"
-import { Button } from "@/components/ui/button"
-import { FilePreview } from "@/components/ui/file-preview"
-import { InterruptPrompt } from "@/components/chat-playground/interrupt-prompt"
+import { cn } from "@/lib/utils";
+import { useAudioRecording } from "@/hooks/use-audio-recording";
+import { useAutosizeTextArea } from "@/hooks/use-autosize-textarea";
+import { AudioVisualizer } from "@/components/ui/audio-visualizer";
+import { Button } from "@/components/ui/button";
+import { FilePreview } from "@/components/ui/file-preview";
+import { InterruptPrompt } from "@/components/chat-playground/interrupt-prompt";
 
 interface MessageInputBaseProps
   extends React.TextareaHTMLAttributes<HTMLTextAreaElement> {
-  value: string
-  submitOnEnter?: boolean
-  stop?: () => void
-  isGenerating: boolean
-  enableInterrupt?: boolean
-  transcribeAudio?: (blob: Blob) => Promise<string>
+  value: string;
+  submitOnEnter?: boolean;
+  stop?: () => void;
+  isGenerating: boolean;
+  enableInterrupt?: boolean;
+  transcribeAudio?: (blob: Blob) => Promise<string>;
 }
 
 interface MessageInputWithoutAttachmentProps extends MessageInputBaseProps {
-  allowAttachments?: false
+  allowAttachments?: false;
 }
 
 interface MessageInputWithAttachmentsProps extends MessageInputBaseProps {
-  allowAttachments: true
-  files: File[] | null
-  setFiles: React.Dispatch<React.SetStateAction<File[] | null>>
+  allowAttachments: true;
+  files: File[] | null;
+  setFiles: React.Dispatch<React.SetStateAction<File[] | null>>;
 }
 
 type MessageInputProps =
   | MessageInputWithoutAttachmentProps
-  | MessageInputWithAttachmentsProps
+  | MessageInputWithAttachmentsProps;
 
 export function MessageInput({
   placeholder = "Ask AI...",
@@ -48,8 +48,8 @@ export function MessageInput({
   transcribeAudio,
   ...props
 }: MessageInputProps) {
-  const [isDragging, setIsDragging] = useState(false)
-  const [showInterruptPrompt, setShowInterruptPrompt] = useState(false)
+  const [isDragging, setIsDragging] = useState(false);
+  const [showInterruptPrompt, setShowInterruptPrompt] = useState(false);
 
   const {
     isListening,
@@ -61,123 +61,124 @@ export function MessageInput({
     stopRecording,
   } = useAudioRecording({
     transcribeAudio,
-    onTranscriptionComplete: (text) => {
-      props.onChange?.({ target: { value: text } } as any)
+    onTranscriptionComplete: text => {
+      props.onChange?.({
+        target: { value: text },
+      } as React.ChangeEvent<HTMLTextAreaElement>);
     },
-  })
+  });
 
   useEffect(() => {
     if (!isGenerating) {
-      setShowInterruptPrompt(false)
+      setShowInterruptPrompt(false);
     }
-  }, [isGenerating])
+  }, [isGenerating]);
 
   const addFiles = (files: File[] | null) => {
     if (props.allowAttachments) {
-      props.setFiles((currentFiles) => {
+      props.setFiles(currentFiles => {
         if (currentFiles === null) {
-          return files
+          return files;
         }
 
         if (files === null) {
-          return currentFiles
+          return currentFiles;
         }
 
-        return [...currentFiles, ...files]
-      })
+        return [...currentFiles, ...files];
+      });
     }
-  }
+  };
 
   const onDragOver = (event: React.DragEvent) => {
-    if (props.allowAttachments !== true) return
-    event.preventDefault()
-    setIsDragging(true)
-  }
+    if (props.allowAttachments !== true) return;
+    event.preventDefault();
+    setIsDragging(true);
+  };
 
   const onDragLeave = (event: React.DragEvent) => {
-    if (props.allowAttachments !== true) return
-    event.preventDefault()
-    setIsDragging(false)
-  }
+    if (props.allowAttachments !== true) return;
+    event.preventDefault();
+    setIsDragging(false);
+  };
 
   const onDrop = (event: React.DragEvent) => {
-    setIsDragging(false)
-    if (props.allowAttachments !== true) return
-    event.preventDefault()
-    const dataTransfer = event.dataTransfer
+    setIsDragging(false);
+    if (props.allowAttachments !== true) return;
+    event.preventDefault();
+    const dataTransfer = event.dataTransfer;
     if (dataTransfer.files.length) {
-      addFiles(Array.from(dataTransfer.files))
+      addFiles(Array.from(dataTransfer.files));
     }
-  }
+  };
 
   const onPaste = (event: React.ClipboardEvent) => {
-    const items = event.clipboardData?.items
-    if (!items) return
+    const items = event.clipboardData?.items;
+    if (!items) return;
 
-    const text = event.clipboardData.getData("text")
+    const text = event.clipboardData.getData("text");
     if (text && text.length > 500 && props.allowAttachments) {
-      event.preventDefault()
-      const blob = new Blob([text], { type: "text/plain" })
+      event.preventDefault();
+      const blob = new Blob([text], { type: "text/plain" });
       const file = new File([blob], "Pasted text", {
         type: "text/plain",
         lastModified: Date.now(),
-      })
-      addFiles([file])
-      return
+      });
+      addFiles([file]);
+      return;
     }
 
     const files = Array.from(items)
-      .map((item) => item.getAsFile())
-      .filter((file) => file !== null)
+      .map(item => item.getAsFile())
+      .filter(file => file !== null);
 
     if (props.allowAttachments && files.length > 0) {
-      addFiles(files)
+      addFiles(files);
     }
-  }
+  };
 
   const onKeyDown = (event: React.KeyboardEvent<HTMLTextAreaElement>) => {
     if (submitOnEnter && event.key === "Enter" && !event.shiftKey) {
-      event.preventDefault()
+      event.preventDefault();
 
       if (isGenerating && stop && enableInterrupt) {
         if (showInterruptPrompt) {
-          stop()
-          setShowInterruptPrompt(false)
-          event.currentTarget.form?.requestSubmit()
+          stop();
+          setShowInterruptPrompt(false);
+          event.currentTarget.form?.requestSubmit();
         } else if (
           props.value ||
           (props.allowAttachments && props.files?.length)
         ) {
-          setShowInterruptPrompt(true)
-          return
+          setShowInterruptPrompt(true);
+          return;
         }
       }
 
-      event.currentTarget.form?.requestSubmit()
+      event.currentTarget.form?.requestSubmit();
     }
 
-    onKeyDownProp?.(event)
-  }
+    onKeyDownProp?.(event);
+  };
 
-  const textAreaRef = useRef<HTMLTextAreaElement>(null)
-  const [textAreaHeight, setTextAreaHeight] = useState<number>(0)
+  const textAreaRef = useRef<HTMLTextAreaElement>(null);
+  const [textAreaHeight, setTextAreaHeight] = useState<number>(0);
 
   useEffect(() => {
     if (textAreaRef.current) {
-      setTextAreaHeight(textAreaRef.current.offsetHeight)
+      setTextAreaHeight(textAreaRef.current.offsetHeight);
     }
-  }, [props.value])
+  }, [props.value]);
 
   const showFileList =
-    props.allowAttachments && props.files && props.files.length > 0
-
+    props.allowAttachments && props.files && props.files.length > 0;
 
   useAutosizeTextArea({
     ref: textAreaRef,
     maxHeight: 240,
     borderWidth: 1,
     dependencies: [props.value, showFileList],
-  })
+  });
 
   return (
     <div
@@ -220,24 +221,24 @@ export function MessageInput({
             <div className="absolute inset-x-3 bottom-0 z-20 overflow-x-scroll py-3">
               <div className="flex space-x-3">
                 <AnimatePresence mode="popLayout">
-                  {props.files?.map((file) => {
+                  {props.files?.map(file => {
                     return (
                       <FilePreview
                         key={file.name + String(file.lastModified)}
                         file={file}
                         onRemove={() => {
-                          props.setFiles((files) => {
-                            if (!files) return null
+                          props.setFiles(files => {
+                            if (!files) return null;
 
                             const filtered = Array.from(files).filter(
-                              (f) => f !== file
-                            )
-                            if (filtered.length === 0) return null
-                            return filtered
-                          })
+                              f => f !== file
+                            );
+                            if (filtered.length === 0) return null;
+                            return filtered;
+                          });
                         }}
                       />
-                    )
+                    );
                   })}
                 </AnimatePresence>
               </div>
@@ -256,8 +257,8 @@ export function MessageInput({
             aria-label="Attach a file"
             disabled={true}
             onClick={async () => {
-              const files = await showFileUploadDialog()
-              addFiles(files)
+              const files = await showFileUploadDialog();
+              addFiles(files);
             }}
           >
             <Paperclip className="h-4 w-4" />
@@ -308,12 +309,12 @@ export function MessageInput({
         onStopRecording={stopRecording}
       />
     </div>
-  )
+  );
 }
-MessageInput.displayName = "MessageInput"
+MessageInput.displayName = "MessageInput";
 
 interface FileUploadOverlayProps {
-  isDragging: boolean
+  isDragging: boolean;
 }
 
 function FileUploadOverlay({ isDragging }: FileUploadOverlayProps) {
@@ -333,29 +334,29 @@ function FileUploadOverlay({ isDragging }: FileUploadOverlayProps) {
         </motion.div>
       )}
     </AnimatePresence>
-  )
+  );
 }
 
 function showFileUploadDialog() {
-  const input = document.createElement("input")
+  const input = document.createElement("input");
 
-  input.type = "file"
-  input.multiple = true
-  input.accept = "*/*"
-  input.click()
+  input.type = "file";
+  input.multiple = true;
+  input.accept = "*/*";
+  input.click();
 
-  return new Promise<File[] | null>((resolve) => {
-    input.onchange = (e) => {
-      const files = (e.currentTarget as HTMLInputElement).files
+  return new Promise<File[] | null>(resolve => {
+    input.onchange = e => {
+      const files = (e.currentTarget as HTMLInputElement).files;
 
       if (files) {
-        resolve(Array.from(files))
-        return
+        resolve(Array.from(files));
+        return;
       }
 
-      resolve(null)
-    }
-  })
+      resolve(null);
+    };
+  });
 }
 
 function TranscribingOverlay() {
@@ -385,12 +386,12 @@ function TranscribingOverlay() {
         Transcribing audio...
       </p>
     </motion.div>
-  )
+  );
 }
 
 interface RecordingPromptProps {
-  isVisible: boolean
-  onStopRecording: () => void
+  isVisible: boolean;
+  onStopRecording: () => void;
 }
 
 function RecordingPrompt({ isVisible, onStopRecording }: RecordingPromptProps) {
@@ -418,15 +419,15 @@ function RecordingPrompt({ isVisible, onStopRecording }: RecordingPromptProps) {
         </motion.div>
       )}
     </AnimatePresence>
-  )
+  );
 }
 
 interface RecordingControlsProps {
-  isRecording: boolean
-  isTranscribing: boolean
-  audioStream: MediaStream | null
-  textAreaHeight: number
-  onStopRecording: () => void
+  isRecording: boolean;
+  isTranscribing: boolean;
+  audioStream: MediaStream | null;
+  textAreaHeight: number;
+  onStopRecording: () => void;
 }
 
 function RecordingControls({
@@ -448,7 +449,7 @@ function RecordingControls({
           onClick={onStopRecording}
         />
       </div>
-    )
+    );
   }
 
   if (isTranscribing) {
@@ -459,8 +460,8 @@ function RecordingControls({
       >
         <TranscribingOverlay />
       </div>
-    )
+    );
   }
 
-  return null
+  return null;
 }
diff --git a/llama_stack/ui/components/chat-playground/message-list.tsx b/llama_stack/ui/components/chat-playground/message-list.tsx
index 5fe8409f4..5e8647748 100644
--- a/llama_stack/ui/components/chat-playground/message-list.tsx
+++ b/llama_stack/ui/components/chat-playground/message-list.tsx
@@ -2,18 +2,18 @@ import {
   ChatMessage,
   type ChatMessageProps,
   type Message,
-} from "@/components/chat-playground/chat-message"
-import { TypingIndicator } from "@/components/chat-playground/typing-indicator"
+} from "@/components/chat-playground/chat-message";
+import { TypingIndicator } from "@/components/chat-playground/typing-indicator";
 
-type AdditionalMessageOptions = Omit<ChatMessageProps, keyof Message>
+type AdditionalMessageOptions = Omit<ChatMessageProps, keyof Message>;
 
 interface MessageListProps {
-  messages: Message[]
-  showTimeStamps?: boolean
-  isTyping?: boolean
+  messages: Message[];
+  showTimeStamps?: boolean;
+  isTyping?: boolean;
   messageOptions?:
     | AdditionalMessageOptions
-    | ((message: Message) => AdditionalMessageOptions)
+    | ((message: Message) => AdditionalMessageOptions);
 }
 
 export function MessageList({
@@ -28,7 +28,7 @@ export function MessageList({
         const additionalOptions =
           typeof messageOptions === "function"
             ? messageOptions(message)
-            : messageOptions
+            : messageOptions;
 
         return (
           <ChatMessage
@@ -37,9 +37,9 @@ export function MessageList({
             {...message}
             {...additionalOptions}
           />
-        )
+        );
       })}
       {isTyping && <TypingIndicator />}
     </div>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/chat-playground/prompt-suggestions.tsx b/llama_stack/ui/components/chat-playground/prompt-suggestions.tsx
index 9afaa4e66..075cce406 100644
--- a/llama_stack/ui/components/chat-playground/prompt-suggestions.tsx
+++ b/llama_stack/ui/components/chat-playground/prompt-suggestions.tsx
@@ -1,7 +1,7 @@
 interface PromptSuggestionsProps {
-  label: string
-  append: (message: { role: "user"; content: string }) => void
-  suggestions: string[]
+  label: string;
+  append: (message: { role: "user"; content: string }) => void;
+  suggestions: string[];
 }
 
 export function PromptSuggestions({
@@ -13,7 +13,7 @@ export function PromptSuggestions({
     <div className="space-y-6">
       <h2 className="text-center text-2xl font-bold">{label}</h2>
       <div className="flex gap-6 text-sm">
-        {suggestions.map((suggestion) => (
+        {suggestions.map(suggestion => (
           <button
             key={suggestion}
             onClick={() => append({ role: "user", content: suggestion })}
@@ -24,5 +24,5 @@ export function PromptSuggestions({
         ))}
       </div>
     </div>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/chat-playground/typing-indicator.tsx b/llama_stack/ui/components/chat-playground/typing-indicator.tsx
index 07055d428..8950c066b 100644
--- a/llama_stack/ui/components/chat-playground/typing-indicator.tsx
+++ b/llama_stack/ui/components/chat-playground/typing-indicator.tsx
@@ -1,4 +1,4 @@
-import { Dot } from "lucide-react"
+import { Dot } from "lucide-react";
 
 export function TypingIndicator() {
   return (
@@ -11,5 +11,5 @@ export function TypingIndicator() {
         </div>
       </div>
     </div>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/layout/app-sidebar.tsx b/llama_stack/ui/components/layout/app-sidebar.tsx
index 2ff106e01..bee3d6a70 100644
--- a/llama_stack/ui/components/layout/app-sidebar.tsx
+++ b/llama_stack/ui/components/layout/app-sidebar.tsx
@@ -56,18 +56,19 @@ const manageItems = [
   },
 ];
 
-const optimizeItems: { title: string; url: string; icon: React.ElementType }[] = [
+const optimizeItems: { title: string; url: string; icon: React.ElementType }[] =
+  [
     {
-        title: "Evaluations",
-        url: "",
-        icon: Compass,
+      title: "Evaluations",
+      url: "",
+      icon: Compass,
     },
     {
-        title: "Fine-tuning",
-        url: "",
-        icon: Settings2,
+      title: "Fine-tuning",
+      url: "",
+      icon: Settings2,
     },
-];
+  ];
 
 interface SidebarItem {
   title: string;
@@ -79,7 +80,7 @@ export function AppSidebar() {
   const pathname = usePathname();
 
   const renderSidebarItems = (items: SidebarItem[]) => {
-    return items.map((item) => {
+    return items.map(item => {
       const isActive = pathname.startsWith(item.url);
       return (
         <SidebarMenuItem key={item.title}>
@@ -88,14 +89,14 @@ export function AppSidebar() {
             className={cn(
               "justify-start",
               isActive &&
-                "bg-gray-200 dark:bg-gray-700 hover:bg-gray-200 dark:hover:bg-gray-700 text-gray-900 dark:text-gray-100",
+                "bg-gray-200 dark:bg-gray-700 hover:bg-gray-200 dark:hover:bg-gray-700 text-gray-900 dark:text-gray-100"
             )}
           >
             <Link href={item.url}>
               <item.icon
                 className={cn(
                   isActive && "text-gray-900 dark:text-gray-100",
-                  "mr-2 h-4 w-4",
+                  "mr-2 h-4 w-4"
                 )}
               />
               <span>{item.title}</span>
@@ -106,46 +107,48 @@ export function AppSidebar() {
     });
   };
 
-return (
-  <Sidebar>
-    <SidebarHeader>
-      <Link href="/">Llama Stack</Link>
-    </SidebarHeader>
-    <SidebarContent>
-      <SidebarGroup>
-        <SidebarGroupLabel>Create</SidebarGroupLabel>
-        <SidebarGroupContent>
-          <SidebarMenu>{renderSidebarItems(createItems)}</SidebarMenu>
-        </SidebarGroupContent>
-      </SidebarGroup>
+  return (
+    <Sidebar>
+      <SidebarHeader>
+        <Link href="/">Llama Stack</Link>
+      </SidebarHeader>
+      <SidebarContent>
+        <SidebarGroup>
+          <SidebarGroupLabel>Create</SidebarGroupLabel>
+          <SidebarGroupContent>
+            <SidebarMenu>{renderSidebarItems(createItems)}</SidebarMenu>
+          </SidebarGroupContent>
+        </SidebarGroup>
 
-      <SidebarGroup>
-        <SidebarGroupLabel>Manage</SidebarGroupLabel>
-        <SidebarGroupContent>
-          <SidebarMenu>{renderSidebarItems(manageItems)}</SidebarMenu>
-        </SidebarGroupContent>
-      </SidebarGroup>
+        <SidebarGroup>
+          <SidebarGroupLabel>Manage</SidebarGroupLabel>
+          <SidebarGroupContent>
+            <SidebarMenu>{renderSidebarItems(manageItems)}</SidebarMenu>
+          </SidebarGroupContent>
+        </SidebarGroup>
 
-      <SidebarGroup>
-        <SidebarGroupLabel>Optimize</SidebarGroupLabel>
-        <SidebarGroupContent>
-          <SidebarMenu>
-            {optimizeItems.map((item) => (
-              <SidebarMenuItem key={item.title}>
-                <SidebarMenuButton
-                  disabled
-                  className="justify-start opacity-60 cursor-not-allowed"
-                >
-                  <item.icon className="mr-2 h-4 w-4" />
-                  <span>{item.title}</span>
-                  <span className="ml-2 text-xs text-gray-500">(Coming Soon)</span>
-                </SidebarMenuButton>
-              </SidebarMenuItem>
-            ))}
-          </SidebarMenu>
-        </SidebarGroupContent>
-      </SidebarGroup>
-    </SidebarContent>
-  </Sidebar>
+        <SidebarGroup>
+          <SidebarGroupLabel>Optimize</SidebarGroupLabel>
+          <SidebarGroupContent>
+            <SidebarMenu>
+              {optimizeItems.map(item => (
+                <SidebarMenuItem key={item.title}>
+                  <SidebarMenuButton
+                    disabled
+                    className="justify-start opacity-60 cursor-not-allowed"
+                  >
+                    <item.icon className="mr-2 h-4 w-4" />
+                    <span>{item.title}</span>
+                    <span className="ml-2 text-xs text-gray-500">
+                      (Coming Soon)
+                    </span>
+                  </SidebarMenuButton>
+                </SidebarMenuItem>
+              ))}
+            </SidebarMenu>
+          </SidebarGroupContent>
+        </SidebarGroup>
+      </SidebarContent>
+    </Sidebar>
   );
 }
diff --git a/llama_stack/ui/components/layout/detail-layout.tsx b/llama_stack/ui/components/layout/detail-layout.tsx
index 3013195a2..ed5edd127 100644
--- a/llama_stack/ui/components/layout/detail-layout.tsx
+++ b/llama_stack/ui/components/layout/detail-layout.tsx
@@ -2,7 +2,7 @@ import React from "react";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { Skeleton } from "@/components/ui/skeleton";
 
-export function DetailLoadingView({ title }: { title: string }) {
+export function DetailLoadingView() {
   return (
     <>
       <Skeleton className="h-8 w-3/4 mb-6" /> {/* Title Skeleton */}
diff --git a/llama_stack/ui/components/logs/logs-table-scroll.test.tsx b/llama_stack/ui/components/logs/logs-table-scroll.test.tsx
index a5c3fde46..9952f750b 100644
--- a/llama_stack/ui/components/logs/logs-table-scroll.test.tsx
+++ b/llama_stack/ui/components/logs/logs-table-scroll.test.tsx
@@ -67,7 +67,7 @@ describe("LogsTable Viewport Loading", () => {
       () => {
         expect(mockLoadMore).toHaveBeenCalled();
       },
-      { timeout: 300 },
+      { timeout: 300 }
     );
 
     expect(mockLoadMore).toHaveBeenCalledTimes(1);
@@ -81,11 +81,11 @@ describe("LogsTable Viewport Loading", () => {
         {...defaultProps}
         status="loading-more"
         onLoadMore={mockLoadMore}
-      />,
+      />
     );
 
     // Wait for possible triggers
-    await new Promise((resolve) => setTimeout(resolve, 300));
+    await new Promise(resolve => setTimeout(resolve, 300));
 
     expect(mockLoadMore).not.toHaveBeenCalled();
   });
@@ -94,15 +94,11 @@ describe("LogsTable Viewport Loading", () => {
     const mockLoadMore = jest.fn();
 
     render(
-      <LogsTable
-        {...defaultProps}
-        status="loading"
-        onLoadMore={mockLoadMore}
-      />,
+      <LogsTable {...defaultProps} status="loading" onLoadMore={mockLoadMore} />
     );
 
     // Wait for possible triggers
-    await new Promise((resolve) => setTimeout(resolve, 300));
+    await new Promise(resolve => setTimeout(resolve, 300));
 
     expect(mockLoadMore).not.toHaveBeenCalled();
   });
@@ -111,18 +107,18 @@ describe("LogsTable Viewport Loading", () => {
     const mockLoadMore = jest.fn();
 
     render(
-      <LogsTable {...defaultProps} hasMore={false} onLoadMore={mockLoadMore} />,
+      <LogsTable {...defaultProps} hasMore={false} onLoadMore={mockLoadMore} />
     );
 
     // Wait for possible triggers
-    await new Promise((resolve) => setTimeout(resolve, 300));
+    await new Promise(resolve => setTimeout(resolve, 300));
 
     expect(mockLoadMore).not.toHaveBeenCalled();
   });
 
   test("sentinel element should not be rendered when loading", () => {
     const { container } = render(
-      <LogsTable {...defaultProps} status="loading-more" />,
+      <LogsTable {...defaultProps} status="loading-more" />
     );
 
     // Check that no sentinel row with height: 1 exists
@@ -132,7 +128,7 @@ describe("LogsTable Viewport Loading", () => {
 
   test("sentinel element should be rendered when not loading and hasMore", () => {
     const { container } = render(
-      <LogsTable {...defaultProps} hasMore={true} status="idle" />,
+      <LogsTable {...defaultProps} hasMore={true} status="idle" />
     );
 
     // Check that sentinel row exists
diff --git a/llama_stack/ui/components/logs/logs-table.test.tsx b/llama_stack/ui/components/logs/logs-table.test.tsx
index 9d129879b..b86cf1c12 100644
--- a/llama_stack/ui/components/logs/logs-table.test.tsx
+++ b/llama_stack/ui/components/logs/logs-table.test.tsx
@@ -70,7 +70,7 @@ describe("LogsTable", () => {
   describe("Loading State", () => {
     test("renders skeleton UI when isLoading is true", () => {
       const { container } = render(
-        <LogsTable {...defaultProps} status="loading" />,
+        <LogsTable {...defaultProps} status="loading" />
       );
 
       // Check for skeleton in the table caption
@@ -78,7 +78,7 @@ describe("LogsTable", () => {
       expect(tableCaption).toBeInTheDocument();
       if (tableCaption) {
         const captionSkeleton = tableCaption.querySelector(
-          '[data-slot="skeleton"]',
+          '[data-slot="skeleton"]'
         );
         expect(captionSkeleton).toBeInTheDocument();
       }
@@ -88,7 +88,7 @@ describe("LogsTable", () => {
       expect(tableBody).toBeInTheDocument();
       if (tableBody) {
         const bodySkeletons = tableBody.querySelectorAll(
-          '[data-slot="skeleton"]',
+          '[data-slot="skeleton"]'
         );
         expect(bodySkeletons.length).toBeGreaterThan(0);
       }
@@ -102,7 +102,7 @@ describe("LogsTable", () => {
 
     test("renders correct number of skeleton rows", () => {
       const { container } = render(
-        <LogsTable {...defaultProps} status="loading" />,
+        <LogsTable {...defaultProps} status="loading" />
       );
 
       const skeletonRows = container.querySelectorAll("tbody tr");
@@ -118,10 +118,10 @@ describe("LogsTable", () => {
           {...defaultProps}
           status="error"
           error={{ name: "Error", message: errorMessage } as Error}
-        />,
+        />
       );
       expect(
-        screen.getByText("Unable to load chat completions"),
+        screen.getByText("Unable to load chat completions")
       ).toBeInTheDocument();
       expect(screen.getByText(errorMessage)).toBeInTheDocument();
     });
@@ -132,29 +132,25 @@ describe("LogsTable", () => {
           {...defaultProps}
           status="error"
           error={{ name: "Error", message: "" } as Error}
-        />,
+        />
       );
       expect(
-        screen.getByText("Unable to load chat completions"),
+        screen.getByText("Unable to load chat completions")
       ).toBeInTheDocument();
       expect(
-        screen.getByText(
-          "An unexpected error occurred while loading the data.",
-        ),
+        screen.getByText("An unexpected error occurred while loading the data.")
       ).toBeInTheDocument();
     });
 
     test("renders default error message when error prop is an object without message", () => {
       render(
-        <LogsTable {...defaultProps} status="error" error={{} as Error} />,
+        <LogsTable {...defaultProps} status="error" error={{} as Error} />
       );
       expect(
-        screen.getByText("Unable to load chat completions"),
+        screen.getByText("Unable to load chat completions")
       ).toBeInTheDocument();
       expect(
-        screen.getByText(
-          "An unexpected error occurred while loading the data.",
-        ),
+        screen.getByText("An unexpected error occurred while loading the data.")
       ).toBeInTheDocument();
     });
 
@@ -164,7 +160,7 @@ describe("LogsTable", () => {
           {...defaultProps}
           status="error"
           error={{ name: "Error", message: "Test error" } as Error}
-        />,
+        />
       );
       const table = screen.queryByRole("table");
       expect(table).not.toBeInTheDocument();
@@ -178,7 +174,7 @@ describe("LogsTable", () => {
           {...defaultProps}
           data={[]}
           emptyMessage="Custom empty message"
-        />,
+        />
       );
       expect(screen.getByText("Custom empty message")).toBeInTheDocument();
 
@@ -214,7 +210,7 @@ describe("LogsTable", () => {
           {...defaultProps}
           data={mockData}
           caption="Custom table caption"
-        />,
+        />
       );
 
       // Table caption
@@ -311,8 +307,8 @@ describe("LogsTable", () => {
       // Verify truncated text is displayed
       const truncatedTexts = screen.getAllByText("This is a ...");
       expect(truncatedTexts).toHaveLength(2); // one for input, one for output
-      truncatedTexts.forEach((textElement) =>
-        expect(textElement).toBeInTheDocument(),
+      truncatedTexts.forEach(textElement =>
+        expect(textElement).toBeInTheDocument()
       );
     });
 
@@ -332,12 +328,12 @@ describe("LogsTable", () => {
 
       // Model name should not be passed to truncateText
       expect(truncateText).not.toHaveBeenCalledWith(
-        "very-long-model-name-that-should-not-be-truncated",
+        "very-long-model-name-that-should-not-be-truncated"
       );
 
       // Full model name should be displayed
       expect(
-        screen.getByText("very-long-model-name-that-should-not-be-truncated"),
+        screen.getByText("very-long-model-name-that-should-not-be-truncated")
       ).toBeInTheDocument();
     });
   });
diff --git a/llama_stack/ui/components/logs/logs-table.tsx b/llama_stack/ui/components/logs/logs-table.tsx
index 3d4e609c7..717b122ca 100644
--- a/llama_stack/ui/components/logs/logs-table.tsx
+++ b/llama_stack/ui/components/logs/logs-table.tsx
@@ -142,7 +142,7 @@ export function LogsTable({
         <Table>
           <TableCaption className="sr-only">{caption}</TableCaption>
           <TableBody>
-            {data.map((row) => (
+            {data.map(row => (
               <TableRow
                 key={row.id}
                 onClick={() => router.push(row.detailPath)}
diff --git a/llama_stack/ui/components/responses/grouping/grouped-items-display.tsx b/llama_stack/ui/components/responses/grouping/grouped-items-display.tsx
index 6ddc0eacc..5eaa93fac 100644
--- a/llama_stack/ui/components/responses/grouping/grouped-items-display.tsx
+++ b/llama_stack/ui/components/responses/grouping/grouped-items-display.tsx
@@ -22,7 +22,7 @@ export function GroupedItemsDisplay({
 
   return (
     <>
-      {groupedItems.map((groupedItem) => {
+      {groupedItems.map(groupedItem => {
         // If this is a function call with an output, render the grouped component
         if (
           groupedItem.outputItem &&
diff --git a/llama_stack/ui/components/responses/hooks/function-call-grouping.ts b/llama_stack/ui/components/responses/hooks/function-call-grouping.ts
index 2994354d5..203cd688f 100644
--- a/llama_stack/ui/components/responses/hooks/function-call-grouping.ts
+++ b/llama_stack/ui/components/responses/hooks/function-call-grouping.ts
@@ -18,7 +18,7 @@ export interface GroupedItem {
  * @returns Array of grouped items with their outputs
  */
 export function useFunctionCallGrouping(
-  items: AnyResponseItem[],
+  items: AnyResponseItem[]
 ): GroupedItem[] {
   return useMemo(() => {
     const groupedItems: GroupedItem[] = [];
diff --git a/llama_stack/ui/components/responses/items/item-renderer.tsx b/llama_stack/ui/components/responses/items/item-renderer.tsx
index 8f65d50c4..5f16d9120 100644
--- a/llama_stack/ui/components/responses/items/item-renderer.tsx
+++ b/llama_stack/ui/components/responses/items/item-renderer.tsx
@@ -52,7 +52,7 @@ export function ItemRenderer({
   // Fallback to generic item for unknown types
   return (
     <GenericItemComponent
-      item={item as any}
+      item={item as Record<string, unknown>}
       index={index}
       keyPrefix={keyPrefix}
     />
diff --git a/llama_stack/ui/components/responses/items/message-item.tsx b/llama_stack/ui/components/responses/items/message-item.tsx
index 5590e4460..68054c48f 100644
--- a/llama_stack/ui/components/responses/items/message-item.tsx
+++ b/llama_stack/ui/components/responses/items/message-item.tsx
@@ -20,7 +20,7 @@ export function MessageItemComponent({
     content = item.content;
   } else if (Array.isArray(item.content)) {
     content = item.content
-      .map((c) => {
+      .map(c => {
         return c.type === "input_text" || c.type === "output_text"
           ? c.text
           : JSON.stringify(c);
diff --git a/llama_stack/ui/components/responses/responses-detail.test.tsx b/llama_stack/ui/components/responses/responses-detail.test.tsx
index f426dc059..c0f348cad 100644
--- a/llama_stack/ui/components/responses/responses-detail.test.tsx
+++ b/llama_stack/ui/components/responses/responses-detail.test.tsx
@@ -18,7 +18,7 @@ describe("ResponseDetailView", () => {
   describe("Loading State", () => {
     test("renders loading skeleton when isLoading is true", () => {
       const { container } = render(
-        <ResponseDetailView {...defaultProps} isLoading={true} />,
+        <ResponseDetailView {...defaultProps} isLoading={true} />
       );
 
       // Check for skeleton elements
@@ -36,13 +36,13 @@ describe("ResponseDetailView", () => {
         <ResponseDetailView
           {...defaultProps}
           error={{ name: "Error", message: errorMessage }}
-        />,
+        />
       );
 
       expect(screen.getByText("Responses Details")).toBeInTheDocument();
       // The error message is split across elements, so we check for parts
       expect(
-        screen.getByText(/Error loading details for ID/),
+        screen.getByText(/Error loading details for ID/)
       ).toBeInTheDocument();
       expect(screen.getByText(/test_id/)).toBeInTheDocument();
       expect(screen.getByText(/Network Error/)).toBeInTheDocument();
@@ -53,11 +53,11 @@ describe("ResponseDetailView", () => {
         <ResponseDetailView
           {...defaultProps}
           error={{ name: "Error", message: "" }}
-        />,
+        />
       );
 
       expect(
-        screen.getByText(/Error loading details for ID/),
+        screen.getByText(/Error loading details for ID/)
       ).toBeInTheDocument();
       expect(screen.getByText(/test_id/)).toBeInTheDocument();
     });
@@ -124,14 +124,14 @@ describe("ResponseDetailView", () => {
       // Check properties - use regex to handle text split across elements
       expect(screen.getByText(/Created/)).toBeInTheDocument();
       expect(
-        screen.getByText(new Date(1710000000 * 1000).toLocaleString()),
+        screen.getByText(new Date(1710000000 * 1000).toLocaleString())
       ).toBeInTheDocument();
 
       // Check for the specific ID label (not Previous Response ID)
       expect(
         screen.getByText((content, element) => {
           return element?.tagName === "STRONG" && content === "ID:";
-        }),
+        })
       ).toBeInTheDocument();
       expect(screen.getByText("resp_123")).toBeInTheDocument();
 
@@ -166,7 +166,7 @@ describe("ResponseDetailView", () => {
       };
 
       render(
-        <ResponseDetailView {...defaultProps} response={minimalResponse} />,
+        <ResponseDetailView {...defaultProps} response={minimalResponse} />
       );
 
       // Should show required properties
@@ -179,7 +179,7 @@ describe("ResponseDetailView", () => {
       expect(screen.queryByText("Top P")).not.toBeInTheDocument();
       expect(screen.queryByText("Parallel Tool Calls")).not.toBeInTheDocument();
       expect(
-        screen.queryByText("Previous Response ID"),
+        screen.queryByText("Previous Response ID")
       ).not.toBeInTheDocument();
     });
 
@@ -196,7 +196,7 @@ describe("ResponseDetailView", () => {
 
       // The error is shown in the properties sidebar, not as a separate "Error" label
       expect(
-        screen.getByText("invalid_request: The request was invalid"),
+        screen.getByText("invalid_request: The request was invalid")
       ).toBeInTheDocument();
     });
   });
@@ -218,7 +218,7 @@ describe("ResponseDetailView", () => {
           {...defaultProps}
           response={mockResponse}
           isLoadingInputItems={true}
-        />,
+        />
       );
 
       // Check for skeleton loading in input items section
@@ -227,7 +227,7 @@ describe("ResponseDetailView", () => {
           {...defaultProps}
           response={mockResponse}
           isLoadingInputItems={true}
-        />,
+        />
       );
 
       const skeletons = container.querySelectorAll('[data-slot="skeleton"]');
@@ -243,16 +243,16 @@ describe("ResponseDetailView", () => {
             name: "Error",
             message: "Failed to load input items",
           }}
-        />,
+        />
       );
 
       expect(
         screen.getByText(
-          "Error loading input items: Failed to load input items",
-        ),
+          "Error loading input items: Failed to load input items"
+        )
       ).toBeInTheDocument();
       expect(
-        screen.getByText("Falling back to response input data."),
+        screen.getByText("Falling back to response input data.")
       ).toBeInTheDocument();
 
       // Should still show fallback input data
@@ -276,7 +276,7 @@ describe("ResponseDetailView", () => {
           {...defaultProps}
           response={mockResponse}
           inputItems={mockInputItems}
-        />,
+        />
       );
 
       // Should show input items data, not response.input
@@ -295,7 +295,7 @@ describe("ResponseDetailView", () => {
           {...defaultProps}
           response={mockResponse}
           inputItems={emptyInputItems}
-        />,
+        />
       );
 
       // Should show fallback input data
@@ -313,7 +313,7 @@ describe("ResponseDetailView", () => {
           {...defaultProps}
           response={responseWithoutInput}
           inputItems={null}
-        />,
+        />
       );
 
       expect(screen.getByText("No input data available.")).toBeInTheDocument();
@@ -443,7 +443,7 @@ describe("ResponseDetailView", () => {
       render(<ResponseDetailView {...defaultProps} response={mockResponse} />);
 
       expect(
-        screen.getByText('input_function({"param": "value"})'),
+        screen.getByText('input_function({"param": "value"})')
       ).toBeInTheDocument();
       expect(screen.getByText("Function Call")).toBeInTheDocument();
     });
@@ -468,7 +468,7 @@ describe("ResponseDetailView", () => {
       render(<ResponseDetailView {...defaultProps} response={mockResponse} />);
 
       expect(
-        screen.getByText("web_search_call(status: completed)"),
+        screen.getByText("web_search_call(status: completed)")
       ).toBeInTheDocument();
       expect(screen.getByText("Function Call")).toBeInTheDocument();
       expect(screen.getByText("(Web Search)")).toBeInTheDocument();
@@ -522,7 +522,7 @@ describe("ResponseDetailView", () => {
       render(<ResponseDetailView {...defaultProps} response={mockResponse} />);
 
       expect(
-        screen.getByText("First output Second output"),
+        screen.getByText("First output Second output")
       ).toBeInTheDocument();
       expect(screen.getByText("Assistant")).toBeInTheDocument();
     });
@@ -549,7 +549,7 @@ describe("ResponseDetailView", () => {
       render(<ResponseDetailView {...defaultProps} response={mockResponse} />);
 
       expect(
-        screen.getByText('search_function({"query": "test"})'),
+        screen.getByText('search_function({"query": "test"})')
       ).toBeInTheDocument();
       expect(screen.getByText("Function Call")).toBeInTheDocument();
     });
@@ -598,7 +598,7 @@ describe("ResponseDetailView", () => {
       render(<ResponseDetailView {...defaultProps} response={mockResponse} />);
 
       expect(
-        screen.getByText("web_search_call(status: completed)"),
+        screen.getByText("web_search_call(status: completed)")
       ).toBeInTheDocument();
       expect(screen.getByText(/Function Call/)).toBeInTheDocument();
       expect(screen.getByText("(Web Search)")).toBeInTheDocument();
@@ -616,7 +616,7 @@ describe("ResponseDetailView", () => {
             type: "unknown_type",
             custom_field: "custom_value",
             data: { nested: "object" },
-          } as any,
+          } as unknown,
         ],
         input: [],
       };
@@ -625,7 +625,7 @@ describe("ResponseDetailView", () => {
 
       // Should show JSON stringified content
       expect(
-        screen.getByText(/custom_field.*custom_value/),
+        screen.getByText(/custom_field.*custom_value/)
       ).toBeInTheDocument();
       expect(screen.getByText("(unknown_type)")).toBeInTheDocument();
     });
@@ -666,7 +666,7 @@ describe("ResponseDetailView", () => {
             role: "assistant",
             call_id: "call_123",
             content: "sunny and warm",
-          } as any, // Using any to bypass the type restriction for this test
+          } as unknown, // Using any to bypass the type restriction for this test
         ],
         input: [],
       };
@@ -676,7 +676,7 @@ describe("ResponseDetailView", () => {
       // Should show the function call and message as separate items (not grouped)
       expect(screen.getByText("Function Call")).toBeInTheDocument();
       expect(
-        screen.getByText('get_weather({"city": "Tokyo"})'),
+        screen.getByText('get_weather({"city": "Tokyo"})')
       ).toBeInTheDocument();
       expect(screen.getByText("Assistant")).toBeInTheDocument();
       expect(screen.getByText("sunny and warm")).toBeInTheDocument();
@@ -706,7 +706,7 @@ describe("ResponseDetailView", () => {
             status: "completed",
             call_id: "call_123",
             output: "sunny and warm",
-          } as any, // Using any to bypass the type restriction for this test
+          } as unknown,
         ],
         input: [],
       };
@@ -717,7 +717,7 @@ describe("ResponseDetailView", () => {
       expect(screen.getByText("Function Call")).toBeInTheDocument();
       expect(screen.getByText("Arguments")).toBeInTheDocument();
       expect(
-        screen.getByText('get_weather({"city": "Tokyo"})'),
+        screen.getByText('get_weather({"city": "Tokyo"})')
       ).toBeInTheDocument();
       // Use getAllByText since there are multiple "Output" elements (card title and output label)
       const outputElements = screen.getAllByText("Output");
diff --git a/llama_stack/ui/components/responses/responses-table.test.tsx b/llama_stack/ui/components/responses/responses-table.test.tsx
index 0338b9151..37eaed543 100644
--- a/llama_stack/ui/components/responses/responses-table.test.tsx
+++ b/llama_stack/ui/components/responses/responses-table.test.tsx
@@ -146,7 +146,7 @@ describe("ResponsesTable", () => {
       expect(tableCaption).toBeInTheDocument();
       if (tableCaption) {
         const captionSkeleton = tableCaption.querySelector(
-          '[data-slot="skeleton"]',
+          '[data-slot="skeleton"]'
         );
         expect(captionSkeleton).toBeInTheDocument();
       }
@@ -156,7 +156,7 @@ describe("ResponsesTable", () => {
       expect(tableBody).toBeInTheDocument();
       if (tableBody) {
         const bodySkeletons = tableBody.querySelectorAll(
-          '[data-slot="skeleton"]',
+          '[data-slot="skeleton"]'
         );
         expect(bodySkeletons.length).toBeGreaterThan(0);
       }
@@ -176,14 +176,14 @@ describe("ResponsesTable", () => {
 
       render(<ResponsesTable {...defaultProps} />);
       expect(
-        screen.getByText("Unable to load chat completions"),
+        screen.getByText("Unable to load chat completions")
       ).toBeInTheDocument();
       expect(screen.getByText(errorMessage)).toBeInTheDocument();
     });
 
     test.each([{ name: "Error", message: "" }, {}])(
       "renders default error message when error has no message",
-      (errorObject) => {
+      errorObject => {
         mockedUsePagination.mockReturnValue({
           data: [],
           status: "error",
@@ -194,14 +194,14 @@ describe("ResponsesTable", () => {
 
         render(<ResponsesTable {...defaultProps} />);
         expect(
-          screen.getByText("Unable to load chat completions"),
+          screen.getByText("Unable to load chat completions")
         ).toBeInTheDocument();
         expect(
           screen.getByText(
-            "An unexpected error occurred while loading the data.",
-          ),
+            "An unexpected error occurred while loading the data."
+          )
         ).toBeInTheDocument();
-      },
+      }
     );
   });
 
@@ -275,7 +275,7 @@ describe("ResponsesTable", () => {
 
       // Table caption
       expect(
-        screen.getByText("A list of your recent responses."),
+        screen.getByText("A list of your recent responses.")
       ).toBeInTheDocument();
 
       // Table headers
@@ -289,14 +289,14 @@ describe("ResponsesTable", () => {
       expect(screen.getByText("Test output")).toBeInTheDocument();
       expect(screen.getByText("llama-test-model")).toBeInTheDocument();
       expect(
-        screen.getByText(new Date(1710000000 * 1000).toLocaleString()),
+        screen.getByText(new Date(1710000000 * 1000).toLocaleString())
       ).toBeInTheDocument();
 
       expect(screen.getByText("Another input")).toBeInTheDocument();
       expect(screen.getByText("Another output")).toBeInTheDocument();
       expect(screen.getByText("llama-another-model")).toBeInTheDocument();
       expect(
-        screen.getByText(new Date(1710001000 * 1000).toLocaleString()),
+        screen.getByText(new Date(1710001000 * 1000).toLocaleString())
       ).toBeInTheDocument();
     });
   });
@@ -487,7 +487,7 @@ describe("ResponsesTable", () => {
 
       render(<ResponsesTable {...defaultProps} />);
       expect(
-        screen.getByText('search_function({"query": "test"})'),
+        screen.getByText('search_function({"query": "test"})')
       ).toBeInTheDocument();
     });
 
@@ -548,7 +548,7 @@ describe("ResponsesTable", () => {
 
       render(<ResponsesTable {...defaultProps} />);
       expect(
-        screen.getByText("web_search_call(status: completed)"),
+        screen.getByText("web_search_call(status: completed)")
       ).toBeInTheDocument();
     });
 
@@ -565,7 +565,7 @@ describe("ResponsesTable", () => {
             id: "unknown_123",
             status: "completed",
             custom_field: "custom_value",
-          } as any,
+          } as unknown,
         ],
         input: [{ type: "message", content: "input" }],
       };
@@ -594,7 +594,7 @@ describe("ResponsesTable", () => {
           {
             type: "unknown_type",
             data: "some data",
-          } as any,
+          } as unknown,
         ],
         input: [{ type: "message", content: "input" }],
       };
@@ -623,7 +623,7 @@ describe("ResponsesTable", () => {
           return typeof text === "string" && text.length > effectiveMaxLength
             ? text.slice(0, effectiveMaxLength) + "..."
             : text;
-        },
+        }
       );
 
       const longInput =
@@ -665,7 +665,7 @@ describe("ResponsesTable", () => {
 
       // The truncated text should be present for both input and output
       const truncatedTexts = screen.getAllByText(
-        longInput.slice(0, 10) + "...",
+        longInput.slice(0, 10) + "..."
       );
       expect(truncatedTexts.length).toBe(2); // one for input, one for output
     });
diff --git a/llama_stack/ui/components/responses/responses-table.tsx b/llama_stack/ui/components/responses/responses-table.tsx
index a3e8c0c15..0c0f8e56b 100644
--- a/llama_stack/ui/components/responses/responses-table.tsx
+++ b/llama_stack/ui/components/responses/responses-table.tsx
@@ -27,7 +27,7 @@ interface ResponsesTableProps {
  * Helper function to convert ResponseListResponse.Data to OpenAIResponse
  */
 const convertResponseListData = (
-  responseData: ResponseListResponse.Data,
+  responseData: ResponseListResponse.Data
 ): OpenAIResponse => {
   return {
     id: responseData.id,
@@ -56,8 +56,8 @@ function getInputText(response: OpenAIResponse): string {
 }
 
 function getOutputText(response: OpenAIResponse): string {
-  const firstMessage = response.output.find((item) =>
-    isMessageItem(item as any),
+  const firstMessage = response.output.find(item =>
+    isMessageItem(item as Record<string, unknown>)
   );
   if (firstMessage) {
     const content = extractContentFromItem(firstMessage as MessageItem);
@@ -66,15 +66,15 @@ function getOutputText(response: OpenAIResponse): string {
     }
   }
 
-  const functionCall = response.output.find((item) =>
-    isFunctionCallItem(item as any),
+  const functionCall = response.output.find(item =>
+    isFunctionCallItem(item as Record<string, unknown>)
   );
   if (functionCall) {
     return formatFunctionCall(functionCall as FunctionCallItem);
   }
 
-  const webSearchCall = response.output.find((item) =>
-    isWebSearchCallItem(item as any),
+  const webSearchCall = response.output.find(item =>
+    isWebSearchCallItem(item as Record<string, unknown>)
   );
   if (webSearchCall) {
     return formatWebSearchCall(webSearchCall as WebSearchCallItem);
@@ -95,7 +95,7 @@ function extractContentFromItem(item: {
   } else if (Array.isArray(item.content)) {
     const textContent = item.content.find(
       (c: ResponseInputMessageContent) =>
-        c.type === "input_text" || c.type === "output_text",
+        c.type === "input_text" || c.type === "output_text"
     );
     return textContent?.text || "";
   }
@@ -131,14 +131,14 @@ export function ResponsesTable({ paginationOptions }: ResponsesTableProps) {
       limit: number;
       model?: string;
       order?: string;
-    },
+    }
   ) => {
     const response = await client.responses.list({
       after: params.after,
       limit: params.limit,
       ...(params.model && { model: params.model }),
       ...(params.order && { order: params.order }),
-    } as any);
+    } as Parameters<typeof client.responses.list>[0]);
 
     const listResponse = response as ResponseListResponse;
 
diff --git a/llama_stack/ui/components/responses/utils/item-types.ts b/llama_stack/ui/components/responses/utils/item-types.ts
index 2bde49119..1c1ca2cb1 100644
--- a/llama_stack/ui/components/responses/utils/item-types.ts
+++ b/llama_stack/ui/components/responses/utils/item-types.ts
@@ -29,7 +29,7 @@ export type AnyResponseItem =
   | FunctionCallOutputItem;
 
 export function isMessageInput(
-  item: ResponseInput,
+  item: ResponseInput
 ): item is ResponseInput & { type: "message" } {
   return item.type === "message";
 }
@@ -39,23 +39,23 @@ export function isMessageItem(item: AnyResponseItem): item is MessageItem {
 }
 
 export function isFunctionCallItem(
-  item: AnyResponseItem,
+  item: AnyResponseItem
 ): item is FunctionCallItem {
   return item.type === "function_call" && "name" in item;
 }
 
 export function isWebSearchCallItem(
-  item: AnyResponseItem,
+  item: AnyResponseItem
 ): item is WebSearchCallItem {
   return item.type === "web_search_call";
 }
 
 export function isFunctionCallOutputItem(
-  item: AnyResponseItem,
+  item: AnyResponseItem
 ): item is FunctionCallOutputItem {
   return (
     item.type === "function_call_output" &&
     "call_id" in item &&
-    typeof (item as any).call_id === "string"
+    typeof (item as Record<string, unknown>).call_id === "string"
   );
 }
diff --git a/llama_stack/ui/components/ui/audio-visualizer.tsx b/llama_stack/ui/components/ui/audio-visualizer.tsx
index e1c23c57b..772ed5eef 100644
--- a/llama_stack/ui/components/ui/audio-visualizer.tsx
+++ b/llama_stack/ui/components/ui/audio-visualizer.tsx
@@ -1,6 +1,6 @@
-"use client"
+"use client";
 
-import { useEffect, useRef } from "react"
+import { useEffect, useRef } from "react";
 
 // Configuration constants for the audio analyzer
 const AUDIO_CONFIG = {
@@ -14,12 +14,12 @@ const AUDIO_CONFIG = {
     MAX_INTENSITY: 255, // Maximum gray value (brighter)
     INTENSITY_RANGE: 155, // MAX_INTENSITY - MIN_INTENSITY
   },
-} as const
+} as const;
 
 interface AudioVisualizerProps {
-  stream: MediaStream | null
-  isRecording: boolean
-  onClick: () => void
+  stream: MediaStream | null;
+  isRecording: boolean;
+  onClick: () => void;
 }
 
 export function AudioVisualizer({
@@ -28,91 +28,91 @@ export function AudioVisualizer({
   onClick,
 }: AudioVisualizerProps) {
   // Refs for managing audio context and animation
-  const canvasRef = useRef<HTMLCanvasElement>(null)
-  const audioContextRef = useRef<AudioContext | null>(null)
-  const analyserRef = useRef<AnalyserNode | null>(null)
-  const animationFrameRef = useRef<number>()
-  const containerRef = useRef<HTMLDivElement>(null)
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const analyserRef = useRef<AnalyserNode | null>(null);
+  const animationFrameRef = useRef<number>();
+  const containerRef = useRef<HTMLDivElement>(null);
 
   // Cleanup function to stop visualization and close audio context
   const cleanup = () => {
     if (animationFrameRef.current) {
-      cancelAnimationFrame(animationFrameRef.current)
+      cancelAnimationFrame(animationFrameRef.current);
     }
     if (audioContextRef.current) {
-      audioContextRef.current.close()
+      audioContextRef.current.close();
     }
-  }
+  };
 
   // Cleanup on unmount
   useEffect(() => {
-    return cleanup
-  }, [])
+    return cleanup;
+  }, []);
 
   // Start or stop visualization based on recording state
   useEffect(() => {
     if (stream && isRecording) {
-      startVisualization()
+      startVisualization();
     } else {
-      cleanup()
+      cleanup();
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [stream, isRecording])
+  }, [stream, isRecording]);
 
   // Handle window resize
   useEffect(() => {
     const handleResize = () => {
       if (canvasRef.current && containerRef.current) {
-        const container = containerRef.current
-        const canvas = canvasRef.current
-        const dpr = window.devicePixelRatio || 1
+        const container = containerRef.current;
+        const canvas = canvasRef.current;
+        const dpr = window.devicePixelRatio || 1;
 
         // Set canvas size based on container and device pixel ratio
-        const rect = container.getBoundingClientRect()
+        const rect = container.getBoundingClientRect();
         // Account for the 2px total margin (1px on each side)
-        canvas.width = (rect.width - 2) * dpr
-        canvas.height = (rect.height - 2) * dpr
+        canvas.width = (rect.width - 2) * dpr;
+        canvas.height = (rect.height - 2) * dpr;
 
         // Scale canvas CSS size to match container minus margins
-        canvas.style.width = `${rect.width - 2}px`
-        canvas.style.height = `${rect.height - 2}px`
+        canvas.style.width = `${rect.width - 2}px`;
+        canvas.style.height = `${rect.height - 2}px`;
       }
-    }
+    };
 
-    window.addEventListener("resize", handleResize)
+    window.addEventListener("resize", handleResize);
     // Initial setup
-    handleResize()
+    handleResize();
 
-    return () => window.removeEventListener("resize", handleResize)
-  }, [])
+    return () => window.removeEventListener("resize", handleResize);
+  }, []);
 
   // Initialize audio context and start visualization
   const startVisualization = async () => {
     try {
-      const audioContext = new AudioContext()
-      audioContextRef.current = audioContext
+      const audioContext = new AudioContext();
+      audioContextRef.current = audioContext;
 
-      const analyser = audioContext.createAnalyser()
-      analyser.fftSize = AUDIO_CONFIG.FFT_SIZE
-      analyser.smoothingTimeConstant = AUDIO_CONFIG.SMOOTHING
-      analyserRef.current = analyser
+      const analyser = audioContext.createAnalyser();
+      analyser.fftSize = AUDIO_CONFIG.FFT_SIZE;
+      analyser.smoothingTimeConstant = AUDIO_CONFIG.SMOOTHING;
+      analyserRef.current = analyser;
 
-      const source = audioContext.createMediaStreamSource(stream!)
-      source.connect(analyser)
+      const source = audioContext.createMediaStreamSource(stream!);
+      source.connect(analyser);
 
-      draw()
+      draw();
     } catch (error) {
-      console.error("Error starting visualization:", error)
+      console.error("Error starting visualization:", error);
     }
-  }
+  };
 
   // Calculate the color intensity based on bar height
   const getBarColor = (normalizedHeight: number) => {
     const intensity =
       Math.floor(normalizedHeight * AUDIO_CONFIG.COLOR.INTENSITY_RANGE) +
-      AUDIO_CONFIG.COLOR.MIN_INTENSITY
-    return `rgb(${intensity}, ${intensity}, ${intensity})`
-  }
+      AUDIO_CONFIG.COLOR.MIN_INTENSITY;
+    return `rgb(${intensity}, ${intensity}, ${intensity})`;
+  };
 
   // Draw a single bar of the visualizer
   const drawBar = (
@@ -123,52 +123,52 @@ export function AudioVisualizer({
     height: number,
     color: string
   ) => {
-    ctx.fillStyle = color
+    ctx.fillStyle = color;
     // Draw upper bar (above center)
-    ctx.fillRect(x, centerY - height, width, height)
+    ctx.fillRect(x, centerY - height, width, height);
     // Draw lower bar (below center)
-    ctx.fillRect(x, centerY, width, height)
-  }
+    ctx.fillRect(x, centerY, width, height);
+  };
 
   // Main drawing function
   const draw = () => {
-    if (!isRecording) return
+    if (!isRecording) return;
 
-    const canvas = canvasRef.current
-    const ctx = canvas?.getContext("2d")
-    if (!canvas || !ctx || !analyserRef.current) return
+    const canvas = canvasRef.current;
+    const ctx = canvas?.getContext("2d");
+    if (!canvas || !ctx || !analyserRef.current) return;
 
-    const dpr = window.devicePixelRatio || 1
-    ctx.scale(dpr, dpr)
+    const dpr = window.devicePixelRatio || 1;
+    ctx.scale(dpr, dpr);
 
-    const analyser = analyserRef.current
-    const bufferLength = analyser.frequencyBinCount
-    const frequencyData = new Uint8Array(bufferLength)
+    const analyser = analyserRef.current;
+    const bufferLength = analyser.frequencyBinCount;
+    const frequencyData = new Uint8Array(bufferLength);
 
     const drawFrame = () => {
-      animationFrameRef.current = requestAnimationFrame(drawFrame)
+      animationFrameRef.current = requestAnimationFrame(drawFrame);
 
       // Get current frequency data
-      analyser.getByteFrequencyData(frequencyData)
+      analyser.getByteFrequencyData(frequencyData);
 
       // Clear canvas - use CSS pixels for clearing
-      ctx.clearRect(0, 0, canvas.width / dpr, canvas.height / dpr)
+      ctx.clearRect(0, 0, canvas.width / dpr, canvas.height / dpr);
 
       // Calculate dimensions in CSS pixels
       const barWidth = Math.max(
         AUDIO_CONFIG.MIN_BAR_WIDTH,
         canvas.width / dpr / bufferLength - AUDIO_CONFIG.BAR_SPACING
-      )
-      const centerY = canvas.height / dpr / 2
-      let x = 0
+      );
+      const centerY = canvas.height / dpr / 2;
+      let x = 0;
 
       // Draw each frequency bar
       for (let i = 0; i < bufferLength; i++) {
-        const normalizedHeight = frequencyData[i] / 255 // Convert to 0-1 range
+        const normalizedHeight = frequencyData[i] / 255; // Convert to 0-1 range
         const barHeight = Math.max(
           AUDIO_CONFIG.MIN_BAR_HEIGHT,
           normalizedHeight * centerY
-        )
+        );
 
         drawBar(
           ctx,
@@ -177,14 +177,14 @@ export function AudioVisualizer({
           barWidth,
           barHeight,
           getBarColor(normalizedHeight)
-        )
+        );
 
-        x += barWidth + AUDIO_CONFIG.BAR_SPACING
+        x += barWidth + AUDIO_CONFIG.BAR_SPACING;
       }
-    }
+    };
 
-    drawFrame()
-  }
+    drawFrame();
+  };
 
   return (
     <div
@@ -194,5 +194,5 @@ export function AudioVisualizer({
     >
       <canvas ref={canvasRef} className="h-full w-full" />
     </div>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/ui/breadcrumb.tsx b/llama_stack/ui/components/ui/breadcrumb.tsx
index f63ae19af..9d88a372a 100644
--- a/llama_stack/ui/components/ui/breadcrumb.tsx
+++ b/llama_stack/ui/components/ui/breadcrumb.tsx
@@ -14,7 +14,7 @@ function BreadcrumbList({ className, ...props }: React.ComponentProps<"ol">) {
       data-slot="breadcrumb-list"
       className={cn(
         "text-muted-foreground flex flex-wrap items-center gap-1.5 text-sm break-words sm:gap-2.5",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/button.tsx b/llama_stack/ui/components/ui/button.tsx
index a2df8dce6..66ab90e53 100644
--- a/llama_stack/ui/components/ui/button.tsx
+++ b/llama_stack/ui/components/ui/button.tsx
@@ -1,8 +1,8 @@
-import * as React from "react"
-import { Slot } from "@radix-ui/react-slot"
-import { cva, type VariantProps } from "class-variance-authority"
+import * as React from "react";
+import { Slot } from "@radix-ui/react-slot";
+import { cva, type VariantProps } from "class-variance-authority";
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils";
 
 const buttonVariants = cva(
   "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
@@ -33,7 +33,7 @@ const buttonVariants = cva(
       size: "default",
     },
   }
-)
+);
 
 function Button({
   className,
@@ -43,9 +43,9 @@ function Button({
   ...props
 }: React.ComponentProps<"button"> &
   VariantProps<typeof buttonVariants> & {
-    asChild?: boolean
+    asChild?: boolean;
   }) {
-  const Comp = asChild ? Slot : "button"
+  const Comp = asChild ? Slot : "button";
 
   return (
     <Comp
@@ -53,7 +53,7 @@ function Button({
       className={cn(buttonVariants({ variant, size, className }))}
       {...props}
     />
-  )
+  );
 }
 
-export { Button, buttonVariants }
+export { Button, buttonVariants };
diff --git a/llama_stack/ui/components/ui/card.tsx b/llama_stack/ui/components/ui/card.tsx
index 113d66c74..93a82d9c1 100644
--- a/llama_stack/ui/components/ui/card.tsx
+++ b/llama_stack/ui/components/ui/card.tsx
@@ -8,7 +8,7 @@ function Card({ className, ...props }: React.ComponentProps<"div">) {
       data-slot="card"
       className={cn(
         "bg-card text-card-foreground flex flex-col gap-6 rounded-xl border py-6 shadow-sm",
-        className,
+        className
       )}
       {...props}
     />
@@ -21,7 +21,7 @@ function CardHeader({ className, ...props }: React.ComponentProps<"div">) {
       data-slot="card-header"
       className={cn(
         "@container/card-header grid auto-rows-min grid-rows-[auto_auto] items-start gap-1.5 px-6 has-data-[slot=card-action]:grid-cols-[1fr_auto] [.border-b]:pb-6",
-        className,
+        className
       )}
       {...props}
     />
@@ -54,7 +54,7 @@ function CardAction({ className, ...props }: React.ComponentProps<"div">) {
       data-slot="card-action"
       className={cn(
         "col-start-2 row-span-2 row-start-1 self-start justify-self-end",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/collapsible.tsx b/llama_stack/ui/components/ui/collapsible.tsx
index ae9fad04a..90935c6b2 100644
--- a/llama_stack/ui/components/ui/collapsible.tsx
+++ b/llama_stack/ui/components/ui/collapsible.tsx
@@ -1,11 +1,11 @@
-"use client"
+"use client";
 
-import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"
+import * as CollapsiblePrimitive from "@radix-ui/react-collapsible";
 
 function Collapsible({
   ...props
 }: React.ComponentProps<typeof CollapsiblePrimitive.Root>) {
-  return <CollapsiblePrimitive.Root data-slot="collapsible" {...props} />
+  return <CollapsiblePrimitive.Root data-slot="collapsible" {...props} />;
 }
 
 function CollapsibleTrigger({
@@ -16,7 +16,7 @@ function CollapsibleTrigger({
       data-slot="collapsible-trigger"
       {...props}
     />
-  )
+  );
 }
 
 function CollapsibleContent({
@@ -27,7 +27,7 @@ function CollapsibleContent({
       data-slot="collapsible-content"
       {...props}
     />
-  )
+  );
 }
 
-export { Collapsible, CollapsibleTrigger, CollapsibleContent }
+export { Collapsible, CollapsibleTrigger, CollapsibleContent };
diff --git a/llama_stack/ui/components/ui/copy-button.tsx b/llama_stack/ui/components/ui/copy-button.tsx
index 51d2ca2d4..433e2474c 100644
--- a/llama_stack/ui/components/ui/copy-button.tsx
+++ b/llama_stack/ui/components/ui/copy-button.tsx
@@ -1,21 +1,21 @@
-"use client"
+"use client";
 
-import { Check, Copy } from "lucide-react"
+import { Check, Copy } from "lucide-react";
 
-import { cn } from "@/lib/utils"
-import { useCopyToClipboard } from "@/hooks/use-copy-to-clipboard"
-import { Button } from "@/components/ui/button"
+import { cn } from "@/lib/utils";
+import { useCopyToClipboard } from "@/hooks/use-copy-to-clipboard";
+import { Button } from "@/components/ui/button";
 
 type CopyButtonProps = {
-  content: string
-  copyMessage?: string
-}
+  content: string;
+  copyMessage?: string;
+};
 
 export function CopyButton({ content, copyMessage }: CopyButtonProps) {
   const { isCopied, handleCopy } = useCopyToClipboard({
     text: content,
     copyMessage,
-  })
+  });
 
   return (
     <Button
@@ -40,5 +40,5 @@ export function CopyButton({ content, copyMessage }: CopyButtonProps) {
         )}
       />
     </Button>
-  )
+  );
 }
diff --git a/llama_stack/ui/components/ui/dropdown-menu.tsx b/llama_stack/ui/components/ui/dropdown-menu.tsx
index 1fc1f4ee3..9cde4a3ca 100644
--- a/llama_stack/ui/components/ui/dropdown-menu.tsx
+++ b/llama_stack/ui/components/ui/dropdown-menu.tsx
@@ -43,7 +43,7 @@ function DropdownMenuContent({
         sideOffset={sideOffset}
         className={cn(
           "bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 max-h-(--radix-dropdown-menu-content-available-height) min-w-[8rem] origin-(--radix-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border p-1 shadow-md",
-          className,
+          className
         )}
         {...props}
       />
@@ -75,7 +75,7 @@ function DropdownMenuItem({
       data-variant={variant}
       className={cn(
         "focus:bg-accent focus:text-accent-foreground data-[variant=destructive]:text-destructive data-[variant=destructive]:focus:bg-destructive/10 dark:data-[variant=destructive]:focus:bg-destructive/20 data-[variant=destructive]:focus:text-destructive data-[variant=destructive]:*:[svg]:!text-destructive [&_svg:not([class*='text-'])]:text-muted-foreground relative flex cursor-default items-center gap-2 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[inset]:pl-8 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
-        className,
+        className
       )}
       {...props}
     />
@@ -93,7 +93,7 @@ function DropdownMenuCheckboxItem({
       data-slot="dropdown-menu-checkbox-item"
       className={cn(
         "focus:bg-accent focus:text-accent-foreground relative flex cursor-default items-center gap-2 rounded-sm py-1.5 pr-2 pl-8 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
-        className,
+        className
       )}
       checked={checked}
       {...props}
@@ -129,7 +129,7 @@ function DropdownMenuRadioItem({
       data-slot="dropdown-menu-radio-item"
       className={cn(
         "focus:bg-accent focus:text-accent-foreground relative flex cursor-default items-center gap-2 rounded-sm py-1.5 pr-2 pl-8 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
-        className,
+        className
       )}
       {...props}
     >
@@ -156,7 +156,7 @@ function DropdownMenuLabel({
       data-inset={inset}
       className={cn(
         "px-2 py-1.5 text-sm font-medium data-[inset]:pl-8",
-        className,
+        className
       )}
       {...props}
     />
@@ -185,7 +185,7 @@ function DropdownMenuShortcut({
       data-slot="dropdown-menu-shortcut"
       className={cn(
         "text-muted-foreground ml-auto text-xs tracking-widest",
-        className,
+        className
       )}
       {...props}
     />
@@ -212,7 +212,7 @@ function DropdownMenuSubTrigger({
       data-inset={inset}
       className={cn(
         "focus:bg-accent focus:text-accent-foreground data-[state=open]:bg-accent data-[state=open]:text-accent-foreground flex cursor-default items-center rounded-sm px-2 py-1.5 text-sm outline-hidden select-none data-[inset]:pl-8",
-        className,
+        className
       )}
       {...props}
     >
@@ -231,7 +231,7 @@ function DropdownMenuSubContent({
       data-slot="dropdown-menu-sub-content"
       className={cn(
         "bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 min-w-[8rem] origin-(--radix-dropdown-menu-content-transform-origin) overflow-hidden rounded-md border p-1 shadow-lg",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/file-preview.tsx b/llama_stack/ui/components/ui/file-preview.tsx
index 8f0ed7da2..5d7dfda7e 100644
--- a/llama_stack/ui/components/ui/file-preview.tsx
+++ b/llama_stack/ui/components/ui/file-preview.tsx
@@ -1,18 +1,18 @@
-"use client"
+"use client";
 
-import React, { useEffect } from "react"
-import { motion } from "framer-motion"
-import { FileIcon, X } from "lucide-react"
+import React, { useEffect } from "react";
+import { motion } from "framer-motion";
+import { FileIcon, X } from "lucide-react";
 
 interface FilePreviewProps {
-  file: File
-  onRemove?: () => void
+  file: File;
+  onRemove?: () => void;
 }
 
 export const FilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
   (props, ref) => {
     if (props.file.type.startsWith("image/")) {
-      return <ImageFilePreview {...props} ref={ref} />
+      return <ImageFilePreview {...props} ref={ref} />;
     }
 
     if (
@@ -20,13 +20,13 @@ export const FilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
       props.file.name.endsWith(".txt") ||
       props.file.name.endsWith(".md")
     ) {
-      return <TextFilePreview {...props} ref={ref} />
+      return <TextFilePreview {...props} ref={ref} />;
     }
 
-    return <GenericFilePreview {...props} ref={ref} />
+    return <GenericFilePreview {...props} ref={ref} />;
   }
-)
-FilePreview.displayName = "FilePreview"
+);
+FilePreview.displayName = "FilePreview";
 
 const ImageFilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
   ({ file, onRemove }, ref) => {
@@ -62,23 +62,23 @@ const ImageFilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
           </button>
         ) : null}
       </motion.div>
-    )
+    );
   }
-)
-ImageFilePreview.displayName = "ImageFilePreview"
+);
+ImageFilePreview.displayName = "ImageFilePreview";
 
 const TextFilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
   ({ file, onRemove }, ref) => {
-    const [preview, setPreview] = React.useState<string>("")
+    const [preview, setPreview] = React.useState<string>("");
 
     useEffect(() => {
-      const reader = new FileReader()
-      reader.onload = (e) => {
-        const text = e.target?.result as string
-        setPreview(text.slice(0, 50) + (text.length > 50 ? "..." : ""))
-      }
-      reader.readAsText(file)
-    }, [file])
+      const reader = new FileReader();
+      reader.onload = e => {
+        const text = e.target?.result as string;
+        setPreview(text.slice(0, 50) + (text.length > 50 ? "..." : ""));
+      };
+      reader.readAsText(file);
+    }, [file]);
 
     return (
       <motion.div
@@ -111,10 +111,10 @@ const TextFilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
           </button>
         ) : null}
       </motion.div>
-    )
+    );
   }
-)
-TextFilePreview.displayName = "TextFilePreview"
+);
+TextFilePreview.displayName = "TextFilePreview";
 
 const GenericFilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
   ({ file, onRemove }, ref) => {
@@ -147,7 +147,7 @@ const GenericFilePreview = React.forwardRef<HTMLDivElement, FilePreviewProps>(
           </button>
         ) : null}
       </motion.div>
-    )
+    );
   }
-)
-GenericFilePreview.displayName = "GenericFilePreview"
+);
+GenericFilePreview.displayName = "GenericFilePreview";
diff --git a/llama_stack/ui/components/ui/input.tsx b/llama_stack/ui/components/ui/input.tsx
index b1a060f50..0316cc455 100644
--- a/llama_stack/ui/components/ui/input.tsx
+++ b/llama_stack/ui/components/ui/input.tsx
@@ -11,7 +11,7 @@ function Input({ className, type, ...props }: React.ComponentProps<"input">) {
         "file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground dark:bg-input/30 border-input flex h-9 w-full min-w-0 rounded-md border bg-transparent px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
         "focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px]",
         "aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/select.tsx b/llama_stack/ui/components/ui/select.tsx
index dcbbc0ca0..c10e42aa5 100644
--- a/llama_stack/ui/components/ui/select.tsx
+++ b/llama_stack/ui/components/ui/select.tsx
@@ -1,27 +1,27 @@
-"use client"
+"use client";
 
-import * as React from "react"
-import * as SelectPrimitive from "@radix-ui/react-select"
-import { CheckIcon, ChevronDownIcon, ChevronUpIcon } from "lucide-react"
+import * as React from "react";
+import * as SelectPrimitive from "@radix-ui/react-select";
+import { CheckIcon, ChevronDownIcon, ChevronUpIcon } from "lucide-react";
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils";
 
 function Select({
   ...props
 }: React.ComponentProps<typeof SelectPrimitive.Root>) {
-  return <SelectPrimitive.Root data-slot="select" {...props} />
+  return <SelectPrimitive.Root data-slot="select" {...props} />;
 }
 
 function SelectGroup({
   ...props
 }: React.ComponentProps<typeof SelectPrimitive.Group>) {
-  return <SelectPrimitive.Group data-slot="select-group" {...props} />
+  return <SelectPrimitive.Group data-slot="select-group" {...props} />;
 }
 
 function SelectValue({
   ...props
 }: React.ComponentProps<typeof SelectPrimitive.Value>) {
-  return <SelectPrimitive.Value data-slot="select-value" {...props} />
+  return <SelectPrimitive.Value data-slot="select-value" {...props} />;
 }
 
 function SelectTrigger({
@@ -30,7 +30,7 @@ function SelectTrigger({
   children,
   ...props
 }: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
-  size?: "sm" | "default"
+  size?: "sm" | "default";
 }) {
   return (
     <SelectPrimitive.Trigger
@@ -47,7 +47,7 @@ function SelectTrigger({
         <ChevronDownIcon className="size-4 opacity-50" />
       </SelectPrimitive.Icon>
     </SelectPrimitive.Trigger>
-  )
+  );
 }
 
 function SelectContent({
@@ -82,7 +82,7 @@ function SelectContent({
         <SelectScrollDownButton />
       </SelectPrimitive.Content>
     </SelectPrimitive.Portal>
-  )
+  );
 }
 
 function SelectLabel({
@@ -95,7 +95,7 @@ function SelectLabel({
       className={cn("text-muted-foreground px-2 py-1.5 text-xs", className)}
       {...props}
     />
-  )
+  );
 }
 
 function SelectItem({
@@ -119,7 +119,7 @@ function SelectItem({
       </span>
       <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
     </SelectPrimitive.Item>
-  )
+  );
 }
 
 function SelectSeparator({
@@ -132,7 +132,7 @@ function SelectSeparator({
       className={cn("bg-border pointer-events-none -mx-1 my-1 h-px", className)}
       {...props}
     />
-  )
+  );
 }
 
 function SelectScrollUpButton({
@@ -150,7 +150,7 @@ function SelectScrollUpButton({
     >
       <ChevronUpIcon className="size-4" />
     </SelectPrimitive.ScrollUpButton>
-  )
+  );
 }
 
 function SelectScrollDownButton({
@@ -168,7 +168,7 @@ function SelectScrollDownButton({
     >
       <ChevronDownIcon className="size-4" />
     </SelectPrimitive.ScrollDownButton>
-  )
+  );
 }
 
 export {
@@ -182,4 +182,4 @@ export {
   SelectSeparator,
   SelectTrigger,
   SelectValue,
-}
+};
diff --git a/llama_stack/ui/components/ui/separator.tsx b/llama_stack/ui/components/ui/separator.tsx
index 06d1380a9..7f8187751 100644
--- a/llama_stack/ui/components/ui/separator.tsx
+++ b/llama_stack/ui/components/ui/separator.tsx
@@ -18,7 +18,7 @@ function Separator({
       orientation={orientation}
       className={cn(
         "bg-border shrink-0 data-[orientation=horizontal]:h-px data-[orientation=horizontal]:w-full data-[orientation=vertical]:h-full data-[orientation=vertical]:w-px",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/sheet.tsx b/llama_stack/ui/components/ui/sheet.tsx
index d30779f4f..6d6efec6a 100644
--- a/llama_stack/ui/components/ui/sheet.tsx
+++ b/llama_stack/ui/components/ui/sheet.tsx
@@ -37,7 +37,7 @@ function SheetOverlay({
       data-slot="sheet-overlay"
       className={cn(
         "data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
-        className,
+        className
       )}
       {...props}
     />
@@ -67,7 +67,7 @@ function SheetContent({
             "data-[state=closed]:slide-out-to-top data-[state=open]:slide-in-from-top inset-x-0 top-0 h-auto border-b",
           side === "bottom" &&
             "data-[state=closed]:slide-out-to-bottom data-[state=open]:slide-in-from-bottom inset-x-0 bottom-0 h-auto border-t",
-          className,
+          className
         )}
         {...props}
       >
diff --git a/llama_stack/ui/components/ui/sidebar.tsx b/llama_stack/ui/components/ui/sidebar.tsx
index f8a0a3ed5..58228e56e 100644
--- a/llama_stack/ui/components/ui/sidebar.tsx
+++ b/llama_stack/ui/components/ui/sidebar.tsx
@@ -85,12 +85,12 @@ function SidebarProvider({
       // This sets the cookie to keep the sidebar state.
       document.cookie = `${SIDEBAR_COOKIE_NAME}=${openState}; path=/; max-age=${SIDEBAR_COOKIE_MAX_AGE}`;
     },
-    [setOpenProp, open],
+    [setOpenProp, open]
   );
 
   // Helper to toggle the sidebar.
   const toggleSidebar = React.useCallback(() => {
-    return isMobile ? setOpenMobile((open) => !open) : setOpen((open) => !open);
+    return isMobile ? setOpenMobile(open => !open) : setOpen(open => !open);
   }, [isMobile, setOpen, setOpenMobile]);
 
   // Adds a keyboard shortcut to toggle the sidebar.
@@ -123,7 +123,7 @@ function SidebarProvider({
       setOpenMobile,
       toggleSidebar,
     }),
-    [state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar],
+    [state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar]
   );
 
   return (
@@ -140,7 +140,7 @@ function SidebarProvider({
           }
           className={cn(
             "group/sidebar-wrapper has-data-[variant=inset]:bg-sidebar flex min-h-svh w-full",
-            className,
+            className
           )}
           {...props}
         >
@@ -171,7 +171,7 @@ function Sidebar({
         data-slot="sidebar"
         className={cn(
           "bg-sidebar text-sidebar-foreground flex h-full w-(--sidebar-width) flex-col",
-          className,
+          className
         )}
         {...props}
       >
@@ -223,7 +223,7 @@ function Sidebar({
           "group-data-[side=right]:rotate-180",
           variant === "floating" || variant === "inset"
             ? "group-data-[collapsible=icon]:w-[calc(var(--sidebar-width-icon)+(--spacing(4)))]"
-            : "group-data-[collapsible=icon]:w-(--sidebar-width-icon)",
+            : "group-data-[collapsible=icon]:w-(--sidebar-width-icon)"
         )}
       />
       <div
@@ -237,7 +237,7 @@ function Sidebar({
           variant === "floating" || variant === "inset"
             ? "p-2 group-data-[collapsible=icon]:w-[calc(var(--sidebar-width-icon)+(--spacing(4))+2px)]"
             : "group-data-[collapsible=icon]:w-(--sidebar-width-icon) group-data-[side=left]:border-r group-data-[side=right]:border-l",
-          className,
+          className
         )}
         {...props}
       >
@@ -267,7 +267,7 @@ function SidebarTrigger({
       variant="ghost"
       size="icon"
       className={cn("size-7", className)}
-      onClick={(event) => {
+      onClick={event => {
         onClick?.(event);
         toggleSidebar();
       }}
@@ -297,7 +297,7 @@ function SidebarRail({ className, ...props }: React.ComponentProps<"button">) {
         "hover:group-data-[collapsible=offcanvas]:bg-sidebar group-data-[collapsible=offcanvas]:translate-x-0 group-data-[collapsible=offcanvas]:after:left-full",
         "[[data-side=left][data-collapsible=offcanvas]_&]:-right-2",
         "[[data-side=right][data-collapsible=offcanvas]_&]:-left-2",
-        className,
+        className
       )}
       {...props}
     />
@@ -311,7 +311,7 @@ function SidebarInset({ className, ...props }: React.ComponentProps<"main">) {
       className={cn(
         "bg-background relative flex w-full flex-1 flex-col",
         "md:peer-data-[variant=inset]:m-2 md:peer-data-[variant=inset]:ml-0 md:peer-data-[variant=inset]:rounded-xl md:peer-data-[variant=inset]:shadow-sm md:peer-data-[variant=inset]:peer-data-[state=collapsed]:ml-2",
-        className,
+        className
       )}
       {...props}
     />
@@ -375,7 +375,7 @@ function SidebarContent({ className, ...props }: React.ComponentProps<"div">) {
       data-sidebar="content"
       className={cn(
         "flex min-h-0 flex-1 flex-col gap-2 overflow-auto group-data-[collapsible=icon]:overflow-hidden",
-        className,
+        className
       )}
       {...props}
     />
@@ -407,7 +407,7 @@ function SidebarGroupLabel({
       className={cn(
         "text-sidebar-foreground/70 ring-sidebar-ring flex h-8 shrink-0 items-center rounded-md px-2 text-xs font-medium outline-hidden transition-[margin,opacity] duration-200 ease-linear focus-visible:ring-2 [&>svg]:size-4 [&>svg]:shrink-0",
         "group-data-[collapsible=icon]:-mt-8 group-data-[collapsible=icon]:opacity-0",
-        className,
+        className
       )}
       {...props}
     />
@@ -430,7 +430,7 @@ function SidebarGroupAction({
         // Increases the hit area of the button on mobile.
         "after:absolute after:-inset-2 md:after:hidden",
         "group-data-[collapsible=icon]:hidden",
-        className,
+        className
       )}
       {...props}
     />
@@ -492,7 +492,7 @@ const sidebarMenuButtonVariants = cva(
       variant: "default",
       size: "default",
     },
-  },
+  }
 );
 
 function SidebarMenuButton({
@@ -570,7 +570,7 @@ function SidebarMenuAction({
         "group-data-[collapsible=icon]:hidden",
         showOnHover &&
           "peer-data-[active=true]/menu-button:text-sidebar-accent-foreground group-focus-within/menu-item:opacity-100 group-hover/menu-item:opacity-100 data-[state=open]:opacity-100 md:opacity-0",
-        className,
+        className
       )}
       {...props}
     />
@@ -592,7 +592,7 @@ function SidebarMenuBadge({
         "peer-data-[size=default]/menu-button:top-1.5",
         "peer-data-[size=lg]/menu-button:top-2.5",
         "group-data-[collapsible=icon]:hidden",
-        className,
+        className
       )}
       {...props}
     />
@@ -645,7 +645,7 @@ function SidebarMenuSub({ className, ...props }: React.ComponentProps<"ul">) {
       className={cn(
         "border-sidebar-border mx-3.5 flex min-w-0 translate-x-px flex-col gap-1 border-l px-2.5 py-0.5",
         "group-data-[collapsible=icon]:hidden",
-        className,
+        className
       )}
       {...props}
     />
@@ -691,7 +691,7 @@ function SidebarMenuSubButton({
         size === "sm" && "text-xs",
         size === "md" && "text-sm",
         "group-data-[collapsible=icon]:hidden",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/sonner.tsx b/llama_stack/ui/components/ui/sonner.tsx
index 957524edb..f1259836a 100644
--- a/llama_stack/ui/components/ui/sonner.tsx
+++ b/llama_stack/ui/components/ui/sonner.tsx
@@ -1,10 +1,10 @@
-"use client"
+"use client";
 
-import { useTheme } from "next-themes"
-import { Toaster as Sonner, ToasterProps } from "sonner"
+import { useTheme } from "next-themes";
+import { Toaster as Sonner, ToasterProps } from "sonner";
 
 const Toaster = ({ ...props }: ToasterProps) => {
-  const { theme = "system" } = useTheme()
+  const { theme = "system" } = useTheme();
 
   return (
     <Sonner
@@ -19,7 +19,7 @@ const Toaster = ({ ...props }: ToasterProps) => {
       }
       {...props}
     />
-  )
-}
+  );
+};
 
-export { Toaster }
+export { Toaster };
diff --git a/llama_stack/ui/components/ui/table.tsx b/llama_stack/ui/components/ui/table.tsx
index 4b3c98ea4..1980f3ad3 100644
--- a/llama_stack/ui/components/ui/table.tsx
+++ b/llama_stack/ui/components/ui/table.tsx
@@ -45,7 +45,7 @@ function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
       data-slot="table-footer"
       className={cn(
         "bg-muted/50 border-t font-medium [&>tr]:last:border-b-0",
-        className,
+        className
       )}
       {...props}
     />
@@ -58,7 +58,7 @@ function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
       data-slot="table-row"
       className={cn(
         "hover:bg-muted/50 data-[state=selected]:bg-muted border-b transition-colors",
-        className,
+        className
       )}
       {...props}
     />
@@ -71,7 +71,7 @@ function TableHead({ className, ...props }: React.ComponentProps<"th">) {
       data-slot="table-head"
       className={cn(
         "text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
-        className,
+        className
       )}
       {...props}
     />
@@ -84,7 +84,7 @@ function TableCell({ className, ...props }: React.ComponentProps<"td">) {
       data-slot="table-cell"
       className={cn(
         "p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
-        className,
+        className
       )}
       {...props}
     />
diff --git a/llama_stack/ui/components/ui/tooltip.tsx b/llama_stack/ui/components/ui/tooltip.tsx
index bf4a342a9..95e0faaf3 100644
--- a/llama_stack/ui/components/ui/tooltip.tsx
+++ b/llama_stack/ui/components/ui/tooltip.tsx
@@ -47,7 +47,7 @@ function TooltipContent({
         sideOffset={sideOffset}
         className={cn(
           "bg-primary text-primary-foreground animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-md px-3 py-1.5 text-xs text-balance",
-          className,
+          className
         )}
         {...props}
       >
diff --git a/llama_stack/ui/components/vector-stores/vector-store-detail.tsx b/llama_stack/ui/components/vector-stores/vector-store-detail.tsx
index 6e26d2e3d..d3d0fa249 100644
--- a/llama_stack/ui/components/vector-stores/vector-store-detail.tsx
+++ b/llama_stack/ui/components/vector-stores/vector-store-detail.tsx
@@ -85,9 +85,9 @@ export function VectorStoreDetailView({
                 </TableRow>
               </TableHeader>
               <TableBody>
-                {files.map((file) => (
+                {files.map(file => (
                   <TableRow key={file.id}>
-                      <TableCell>
+                    <TableCell>
                       <Button
                         variant="link"
                         className="p-0 h-auto font-mono text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300"
diff --git a/llama_stack/ui/e2e/logs-table-scroll.spec.ts b/llama_stack/ui/e2e/logs-table-scroll.spec.ts
index 081e6d426..345fea599 100644
--- a/llama_stack/ui/e2e/logs-table-scroll.spec.ts
+++ b/llama_stack/ui/e2e/logs-table-scroll.spec.ts
@@ -45,7 +45,7 @@ test.describe("LogsTable Scroll and Progressive Loading", () => {
     const scrollContainer = page.locator("div.overflow-auto").first();
 
     // Scroll to near the bottom
-    await scrollContainer.evaluate((element) => {
+    await scrollContainer.evaluate(element => {
       element.scrollTop = element.scrollHeight - element.clientHeight - 100;
     });
 
diff --git a/llama_stack/ui/eslint.config.mjs b/llama_stack/ui/eslint.config.mjs
index c85fb67c4..354c3bb15 100644
--- a/llama_stack/ui/eslint.config.mjs
+++ b/llama_stack/ui/eslint.config.mjs
@@ -10,7 +10,13 @@ const compat = new FlatCompat({
 });
 
 const eslintConfig = [
-  ...compat.extends("next/core-web-vitals", "next/typescript"),
+  ...compat.extends("next/core-web-vitals", "next/typescript", "prettier"),
+  ...compat.plugins("prettier"),
+  {
+    rules: {
+      "prettier/prettier": "error",
+    },
+  },
 ];
 
 export default eslintConfig;
diff --git a/llama_stack/ui/hooks/use-audio-recording.ts b/llama_stack/ui/hooks/use-audio-recording.ts
index dd58ce6e7..4d08837e9 100644
--- a/llama_stack/ui/hooks/use-audio-recording.ts
+++ b/llama_stack/ui/hooks/use-audio-recording.ts
@@ -1,85 +1,85 @@
-import { useEffect, useRef, useState } from "react"
+import { useEffect, useRef, useState } from "react";
 
-import { recordAudio } from "@/lib/audio-utils"
+import { recordAudio } from "@/lib/audio-utils";
 
 interface UseAudioRecordingOptions {
-  transcribeAudio?: (blob: Blob) => Promise<string>
-  onTranscriptionComplete?: (text: string) => void
+  transcribeAudio?: (blob: Blob) => Promise<string>;
+  onTranscriptionComplete?: (text: string) => void;
 }
 
 export function useAudioRecording({
   transcribeAudio,
   onTranscriptionComplete,
 }: UseAudioRecordingOptions) {
-  const [isListening, setIsListening] = useState(false)
-  const [isSpeechSupported, setIsSpeechSupported] = useState(!!transcribeAudio)
-  const [isRecording, setIsRecording] = useState(false)
-  const [isTranscribing, setIsTranscribing] = useState(false)
-  const [audioStream, setAudioStream] = useState<MediaStream | null>(null)
-  const activeRecordingRef = useRef<any>(null)
+  const [isListening, setIsListening] = useState(false);
+  const [isSpeechSupported, setIsSpeechSupported] = useState(!!transcribeAudio);
+  const [isRecording, setIsRecording] = useState(false);
+  const [isTranscribing, setIsTranscribing] = useState(false);
+  const [audioStream, setAudioStream] = useState<MediaStream | null>(null);
+  const activeRecordingRef = useRef<any>(null);
 
   useEffect(() => {
     const checkSpeechSupport = async () => {
       const hasMediaDevices = !!(
         navigator.mediaDevices && navigator.mediaDevices.getUserMedia
-      )
-      setIsSpeechSupported(hasMediaDevices && !!transcribeAudio)
-    }
+      );
+      setIsSpeechSupported(hasMediaDevices && !!transcribeAudio);
+    };
 
-    checkSpeechSupport()
-  }, [transcribeAudio])
+    checkSpeechSupport();
+  }, [transcribeAudio]);
 
   const stopRecording = async () => {
-    setIsRecording(false)
-    setIsTranscribing(true)
+    setIsRecording(false);
+    setIsTranscribing(true);
     try {
       // First stop the recording to get the final blob
-      recordAudio.stop()
+      recordAudio.stop();
       // Wait for the recording promise to resolve with the final blob
-      const recording = await activeRecordingRef.current
+      const recording = await activeRecordingRef.current;
       if (transcribeAudio) {
-        const text = await transcribeAudio(recording)
-        onTranscriptionComplete?.(text)
+        const text = await transcribeAudio(recording);
+        onTranscriptionComplete?.(text);
       }
     } catch (error) {
-      console.error("Error transcribing audio:", error)
+      console.error("Error transcribing audio:", error);
     } finally {
-      setIsTranscribing(false)
-      setIsListening(false)
+      setIsTranscribing(false);
+      setIsListening(false);
       if (audioStream) {
-        audioStream.getTracks().forEach((track) => track.stop())
-        setAudioStream(null)
+        audioStream.getTracks().forEach(track => track.stop());
+        setAudioStream(null);
       }
-      activeRecordingRef.current = null
+      activeRecordingRef.current = null;
     }
-  }
+  };
 
   const toggleListening = async () => {
     if (!isListening) {
       try {
-        setIsListening(true)
-        setIsRecording(true)
+        setIsListening(true);
+        setIsRecording(true);
         // Get audio stream first
         const stream = await navigator.mediaDevices.getUserMedia({
           audio: true,
-        })
-        setAudioStream(stream)
+        });
+        setAudioStream(stream);
 
         // Start recording with the stream
-        activeRecordingRef.current = recordAudio(stream)
+        activeRecordingRef.current = recordAudio(stream);
       } catch (error) {
-        console.error("Error recording audio:", error)
-        setIsListening(false)
-        setIsRecording(false)
+        console.error("Error recording audio:", error);
+        setIsListening(false);
+        setIsRecording(false);
         if (audioStream) {
-          audioStream.getTracks().forEach((track) => track.stop())
-          setAudioStream(null)
+          audioStream.getTracks().forEach(track => track.stop());
+          setAudioStream(null);
         }
       }
     } else {
-      await stopRecording()
+      await stopRecording();
     }
-  }
+  };
 
   return {
     isListening,
@@ -89,5 +89,5 @@ export function useAudioRecording({
     audioStream,
     toggleListening,
     stopRecording,
-  }
+  };
 }
diff --git a/llama_stack/ui/hooks/use-auto-scroll.ts b/llama_stack/ui/hooks/use-auto-scroll.ts
index 4d22c2cef..170aca688 100644
--- a/llama_stack/ui/hooks/use-auto-scroll.ts
+++ b/llama_stack/ui/hooks/use-auto-scroll.ts
@@ -1,67 +1,67 @@
-import { useEffect, useRef, useState } from "react"
+import { useEffect, useRef, useState } from "react";
 
 // How many pixels from the bottom of the container to enable auto-scroll
-const ACTIVATION_THRESHOLD = 50
+const ACTIVATION_THRESHOLD = 50;
 // Minimum pixels of scroll-up movement required to disable auto-scroll
-const MIN_SCROLL_UP_THRESHOLD = 10
+const MIN_SCROLL_UP_THRESHOLD = 10;
 
 export function useAutoScroll(dependencies: React.DependencyList) {
-  const containerRef = useRef<HTMLDivElement | null>(null)
-  const previousScrollTop = useRef<number | null>(null)
-  const [shouldAutoScroll, setShouldAutoScroll] = useState(true)
+  const containerRef = useRef<HTMLDivElement | null>(null);
+  const previousScrollTop = useRef<number | null>(null);
+  const [shouldAutoScroll, setShouldAutoScroll] = useState(true);
 
   const scrollToBottom = () => {
     if (containerRef.current) {
-      containerRef.current.scrollTop = containerRef.current.scrollHeight
+      containerRef.current.scrollTop = containerRef.current.scrollHeight;
     }
-  }
+  };
 
   const handleScroll = () => {
     if (containerRef.current) {
-      const { scrollTop, scrollHeight, clientHeight } = containerRef.current
+      const { scrollTop, scrollHeight, clientHeight } = containerRef.current;
 
       const distanceFromBottom = Math.abs(
         scrollHeight - scrollTop - clientHeight
-      )
+      );
 
       const isScrollingUp = previousScrollTop.current
         ? scrollTop < previousScrollTop.current
-        : false
+        : false;
 
       const scrollUpDistance = previousScrollTop.current
         ? previousScrollTop.current - scrollTop
-        : 0
+        : 0;
 
       const isDeliberateScrollUp =
-        isScrollingUp && scrollUpDistance > MIN_SCROLL_UP_THRESHOLD
+        isScrollingUp && scrollUpDistance > MIN_SCROLL_UP_THRESHOLD;
 
       if (isDeliberateScrollUp) {
-        setShouldAutoScroll(false)
+        setShouldAutoScroll(false);
       } else {
-        const isScrolledToBottom = distanceFromBottom < ACTIVATION_THRESHOLD
-        setShouldAutoScroll(isScrolledToBottom)
+        const isScrolledToBottom = distanceFromBottom < ACTIVATION_THRESHOLD;
+        setShouldAutoScroll(isScrolledToBottom);
       }
 
-      previousScrollTop.current = scrollTop
+      previousScrollTop.current = scrollTop;
     }
-  }
+  };
 
   const handleTouchStart = () => {
-    setShouldAutoScroll(false)
-  }
+    setShouldAutoScroll(false);
+  };
 
   useEffect(() => {
     if (containerRef.current) {
-      previousScrollTop.current = containerRef.current.scrollTop
+      previousScrollTop.current = containerRef.current.scrollTop;
     }
-  }, [])
+  }, []);
 
   useEffect(() => {
     if (shouldAutoScroll) {
-      scrollToBottom()
+      scrollToBottom();
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, dependencies)
+  }, dependencies);
 
   return {
     containerRef,
@@ -69,5 +69,5 @@ export function useAutoScroll(dependencies: React.DependencyList) {
     handleScroll,
     shouldAutoScroll,
     handleTouchStart,
-  }
+  };
 }
diff --git a/llama_stack/ui/hooks/use-autosize-textarea.ts b/llama_stack/ui/hooks/use-autosize-textarea.ts
index a0a36bb02..a38359033 100644
--- a/llama_stack/ui/hooks/use-autosize-textarea.ts
+++ b/llama_stack/ui/hooks/use-autosize-textarea.ts
@@ -1,10 +1,10 @@
-import { useLayoutEffect, useRef } from "react"
+import { useLayoutEffect, useRef } from "react";
 
 interface UseAutosizeTextAreaProps {
-  ref: React.RefObject<HTMLTextAreaElement | null>
-  maxHeight?: number
-  borderWidth?: number
-  dependencies: React.DependencyList
+  ref: React.RefObject<HTMLTextAreaElement | null>;
+  maxHeight?: number;
+  borderWidth?: number;
+  dependencies: React.DependencyList;
 }
 
 export function useAutosizeTextArea({
@@ -13,27 +13,27 @@ export function useAutosizeTextArea({
   borderWidth = 0,
   dependencies,
 }: UseAutosizeTextAreaProps) {
-  const originalHeight = useRef<number | null>(null)
+  const originalHeight = useRef<number | null>(null);
 
   useLayoutEffect(() => {
-    if (!ref.current) return
+    if (!ref.current) return;
 
-    const currentRef = ref.current
-    const borderAdjustment = borderWidth * 2
+    const currentRef = ref.current;
+    const borderAdjustment = borderWidth * 2;
 
     if (originalHeight.current === null) {
-      originalHeight.current = currentRef.scrollHeight - borderAdjustment
+      originalHeight.current = currentRef.scrollHeight - borderAdjustment;
     }
 
-    currentRef.style.removeProperty("height")
-    const scrollHeight = currentRef.scrollHeight
+    currentRef.style.removeProperty("height");
+    const scrollHeight = currentRef.scrollHeight;
 
     // Make sure we don't go over maxHeight
-    const clampedToMax = Math.min(scrollHeight, maxHeight)
+    const clampedToMax = Math.min(scrollHeight, maxHeight);
     // Make sure we don't go less than the original height
-    const clampedToMin = Math.max(clampedToMax, originalHeight.current)
+    const clampedToMin = Math.max(clampedToMax, originalHeight.current);
 
-    currentRef.style.height = `${clampedToMin + borderAdjustment}px`
+    currentRef.style.height = `${clampedToMin + borderAdjustment}px`;
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [maxHeight, ref, ...dependencies])
+  }, [maxHeight, ref, ...dependencies]);
 }
diff --git a/llama_stack/ui/hooks/use-copy-to-clipboard.ts b/llama_stack/ui/hooks/use-copy-to-clipboard.ts
index e2468d811..90043c4a0 100644
--- a/llama_stack/ui/hooks/use-copy-to-clipboard.ts
+++ b/llama_stack/ui/hooks/use-copy-to-clipboard.ts
@@ -1,36 +1,36 @@
-import { useCallback, useRef, useState } from "react"
-import { toast } from "sonner"
+import { useCallback, useRef, useState } from "react";
+import { toast } from "sonner";
 
 type UseCopyToClipboardProps = {
-  text: string
-  copyMessage?: string
-}
+  text: string;
+  copyMessage?: string;
+};
 
 export function useCopyToClipboard({
   text,
   copyMessage = "Copied to clipboard!",
 }: UseCopyToClipboardProps) {
-  const [isCopied, setIsCopied] = useState(false)
-  const timeoutRef = useRef<NodeJS.Timeout | null>(null)
+  const [isCopied, setIsCopied] = useState(false);
+  const timeoutRef = useRef<NodeJS.Timeout | null>(null);
 
   const handleCopy = useCallback(() => {
     navigator.clipboard
       .writeText(text)
       .then(() => {
-        toast.success(copyMessage)
-        setIsCopied(true)
+        toast.success(copyMessage);
+        setIsCopied(true);
         if (timeoutRef.current) {
-          clearTimeout(timeoutRef.current)
-          timeoutRef.current = null
+          clearTimeout(timeoutRef.current);
+          timeoutRef.current = null;
         }
         timeoutRef.current = setTimeout(() => {
-          setIsCopied(false)
-        }, 2000)
+          setIsCopied(false);
+        }, 2000);
       })
       .catch(() => {
-        toast.error("Failed to copy to clipboard.")
-      })
-  }, [text, copyMessage])
+        toast.error("Failed to copy to clipboard.");
+      });
+  }, [text, copyMessage]);
 
-  return { isCopied, handleCopy }
+  return { isCopied, handleCopy };
 }
diff --git a/llama_stack/ui/hooks/use-infinite-scroll.ts b/llama_stack/ui/hooks/use-infinite-scroll.ts
index 08a64a899..889c3f9fb 100644
--- a/llama_stack/ui/hooks/use-infinite-scroll.ts
+++ b/llama_stack/ui/hooks/use-infinite-scroll.ts
@@ -20,7 +20,7 @@ interface UseInfiniteScrollOptions {
  */
 export function useInfiniteScroll(
   onLoadMore: (() => void) | undefined,
-  options: UseInfiniteScrollOptions = {},
+  options: UseInfiniteScrollOptions = {}
 ) {
   const { enabled = true, threshold = 0.1, rootMargin = "100px" } = options;
   const sentinelRef = useRef<HTMLTableRowElement>(null);
@@ -29,7 +29,7 @@ export function useInfiniteScroll(
     if (!onLoadMore || !enabled) return;
 
     const observer = new IntersectionObserver(
-      (entries) => {
+      entries => {
         const [entry] = entries;
         if (entry.isIntersecting) {
           onLoadMore();
@@ -38,7 +38,7 @@ export function useInfiniteScroll(
       {
         threshold,
         rootMargin,
-      },
+      }
     );
 
     const sentinel = sentinelRef.current;
diff --git a/llama_stack/ui/hooks/use-mobile.ts b/llama_stack/ui/hooks/use-mobile.ts
index a93d58393..48fab93c0 100644
--- a/llama_stack/ui/hooks/use-mobile.ts
+++ b/llama_stack/ui/hooks/use-mobile.ts
@@ -4,7 +4,7 @@ const MOBILE_BREAKPOINT = 768;
 
 export function useIsMobile() {
   const [isMobile, setIsMobile] = React.useState<boolean | undefined>(
-    undefined,
+    undefined
   );
 
   React.useEffect(() => {
diff --git a/llama_stack/ui/hooks/use-pagination.ts b/llama_stack/ui/hooks/use-pagination.ts
index 58847ece5..9fa4fa338 100644
--- a/llama_stack/ui/hooks/use-pagination.ts
+++ b/llama_stack/ui/hooks/use-pagination.ts
@@ -38,7 +38,7 @@ interface UsePaginationParams<T> extends UsePaginationOptions {
       limit: number;
       model?: string;
       order?: string;
-    },
+    }
   ) => Promise<PaginationResponse<T>>;
   errorMessagePrefix: string;
   enabled?: boolean;
@@ -81,7 +81,7 @@ export function usePagination<T>({
       const fetchLimit = targetRows || limit;
 
       try {
-        setState((prev) => ({
+        setState(prev => ({
           ...prev,
           status: isInitialLoad ? "loading" : "loading-more",
           error: null,
@@ -94,7 +94,7 @@ export function usePagination<T>({
           ...(order && { order }),
         });
 
-        setState((prev) => ({
+        setState(prev => ({
           ...prev,
           data: isInitialLoad
             ? response.data
@@ -124,14 +124,14 @@ export function usePagination<T>({
             ? new Error(`${errorMessage} ${err.message}`)
             : new Error(errorMessage);
 
-        setState((prev) => ({
+        setState(prev => ({
           ...prev,
           error,
           status: "error",
         }));
       }
     },
-    [limit, model, order, fetchFunction, errorMessagePrefix, client, router],
+    [limit, model, order, fetchFunction, errorMessagePrefix, client, router]
   );
 
   /**
diff --git a/llama_stack/ui/lib/audio-utils.ts b/llama_stack/ui/lib/audio-utils.ts
index b9ad9a3ef..24c4becfd 100644
--- a/llama_stack/ui/lib/audio-utils.ts
+++ b/llama_stack/ui/lib/audio-utils.ts
@@ -1,50 +1,50 @@
 type RecordAudioType = {
-  (stream: MediaStream): Promise<Blob>
-  stop: () => void
-  currentRecorder?: MediaRecorder
-}
+  (stream: MediaStream): Promise<Blob>;
+  stop: () => void;
+  currentRecorder?: MediaRecorder;
+};
 
 export const recordAudio = (function (): RecordAudioType {
   const func = async function recordAudio(stream: MediaStream): Promise<Blob> {
     try {
       const mediaRecorder = new MediaRecorder(stream, {
         mimeType: "audio/webm;codecs=opus",
-      })
-      const audioChunks: Blob[] = []
+      });
+      const audioChunks: Blob[] = [];
 
       return new Promise((resolve, reject) => {
-        mediaRecorder.ondataavailable = (event) => {
+        mediaRecorder.ondataavailable = event => {
           if (event.data.size > 0) {
-            audioChunks.push(event.data)
+            audioChunks.push(event.data);
           }
-        }
+        };
 
         mediaRecorder.onstop = () => {
-          const audioBlob = new Blob(audioChunks, { type: "audio/webm" })
-          resolve(audioBlob)
-        }
+          const audioBlob = new Blob(audioChunks, { type: "audio/webm" });
+          resolve(audioBlob);
+        };
 
         mediaRecorder.onerror = () => {
-          reject(new Error("MediaRecorder error occurred"))
-        }
+          reject(new Error("MediaRecorder error occurred"));
+        };
 
-        mediaRecorder.start(1000)
-        ;(func as RecordAudioType).currentRecorder = mediaRecorder
-      })
+        mediaRecorder.start(1000);
+        (func as RecordAudioType).currentRecorder = mediaRecorder;
+      });
     } catch (error) {
       const errorMessage =
-        error instanceof Error ? error.message : "Unknown error occurred"
-      throw new Error("Failed to start recording: " + errorMessage)
+        error instanceof Error ? error.message : "Unknown error occurred";
+      throw new Error("Failed to start recording: " + errorMessage);
     }
-  }
+  };
 
-  ;(func as RecordAudioType).stop = () => {
-    const recorder = (func as RecordAudioType).currentRecorder
+  (func as RecordAudioType).stop = () => {
+    const recorder = (func as RecordAudioType).currentRecorder;
     if (recorder && recorder.state !== "inactive") {
-      recorder.stop()
+      recorder.stop();
     }
-    delete (func as RecordAudioType).currentRecorder
-  }
+    delete (func as RecordAudioType).currentRecorder;
+  };
 
-  return func as RecordAudioType
-})()
+  return func as RecordAudioType;
+})();
diff --git a/llama_stack/ui/lib/config-validator.ts b/llama_stack/ui/lib/config-validator.ts
index 19f4397b8..0020942f9 100644
--- a/llama_stack/ui/lib/config-validator.ts
+++ b/llama_stack/ui/lib/config-validator.ts
@@ -27,19 +27,19 @@ export function validateServerConfig() {
       !optionalConfigs.GITHUB_CLIENT_SECRET
     ) {
       console.log(
-        "\n📝 GitHub OAuth not configured (authentication features disabled)",
+        "\n📝 GitHub OAuth not configured (authentication features disabled)"
       );
       console.log("   To enable GitHub OAuth:");
       console.log("   1. Go to https://github.com/settings/applications/new");
       console.log(
-        "   2. Set Application name: Llama Stack UI (or your preferred name)",
+        "   2. Set Application name: Llama Stack UI (or your preferred name)"
       );
       console.log("   3. Set Homepage URL: http://localhost:8322");
       console.log(
-        "   4. Set Authorization callback URL: http://localhost:8322/api/auth/callback/github",
+        "   4. Set Authorization callback URL: http://localhost:8322/api/auth/callback/github"
       );
       console.log(
-        "   5. Create the app and copy the Client ID and Client Secret",
+        "   5. Create the app and copy the Client ID and Client Secret"
       );
       console.log("   6. Add them to your .env.local file:");
       console.log("      GITHUB_CLIENT_ID=your_client_id");
diff --git a/llama_stack/ui/lib/contents-api.ts b/llama_stack/ui/lib/contents-api.ts
index b8fcdb1a2..f4920f3db 100644
--- a/llama_stack/ui/lib/contents-api.ts
+++ b/llama_stack/ui/lib/contents-api.ts
@@ -11,7 +11,7 @@ export interface VectorStoreContentItem {
   vector_store_id: string;
   file_id: string;
   content: VectorStoreContent;
-  metadata: Record<string, any>;
+  metadata: Record<string, unknown>;
   embedding?: number[];
 }
 
@@ -32,11 +32,18 @@ export interface VectorStoreListContentsResponse {
 export class ContentsAPI {
   constructor(private client: LlamaStackClient) {}
 
-  async getFileContents(vectorStoreId: string, fileId: string): Promise<VectorStoreContentsResponse> {
+  async getFileContents(
+    vectorStoreId: string,
+    fileId: string
+  ): Promise<VectorStoreContentsResponse> {
     return this.client.vectorStores.files.content(vectorStoreId, fileId);
   }
 
-  async getContent(vectorStoreId: string, fileId: string, contentId: string): Promise<VectorStoreContentItem> {
+  async getContent(
+    vectorStoreId: string,
+    fileId: string,
+    contentId: string
+  ): Promise<VectorStoreContentItem> {
     const contentsResponse = await this.listContents(vectorStoreId, fileId);
     const targetContent = contentsResponse.data.find(c => c.id === contentId);
 
@@ -47,16 +54,11 @@ export class ContentsAPI {
     return targetContent;
   }
 
-  async updateContent(
-    vectorStoreId: string,
-    fileId: string,
-    contentId: string,
-    updates: { content?: string; metadata?: Record<string, any> }
-  ): Promise<VectorStoreContentItem> {
+  async updateContent(): Promise<VectorStoreContentItem> {
     throw new Error("Individual content updates not yet implemented in API");
   }
 
-  async deleteContent(vectorStoreId: string, fileId: string, contentId: string): Promise<VectorStoreContentDeleteResponse> {
+  async deleteContent(): Promise<VectorStoreContentDeleteResponse> {
     throw new Error("Individual content deletion not yet implemented in API");
   }
 
@@ -70,18 +72,27 @@ export class ContentsAPI {
       before?: string;
     }
   ): Promise<VectorStoreListContentsResponse> {
-    const fileContents = await this.client.vectorStores.files.content(vectorStoreId, fileId);
+    const fileContents = await this.client.vectorStores.files.content(
+      vectorStoreId,
+      fileId
+    );
     const contentItems: VectorStoreContentItem[] = [];
 
     fileContents.content.forEach((content, contentIndex) => {
-      const rawContent = content as any;
+      const rawContent = content as Record<string, unknown>;
 
       // Extract actual fields from the API response
       const embedding = rawContent.embedding || undefined;
-      const created_timestamp = rawContent.created_timestamp || rawContent.created_at || Date.now() / 1000;
+      const created_timestamp =
+        rawContent.created_timestamp ||
+        rawContent.created_at ||
+        Date.now() / 1000;
       const chunkMetadata = rawContent.chunk_metadata || {};
-      const contentId = rawContent.chunk_metadata?.chunk_id || rawContent.id || `content_${fileId}_${contentIndex}`;
-      const objectType = rawContent.object || 'vector_store.file.content';
+      const contentId =
+        rawContent.chunk_metadata?.chunk_id ||
+        rawContent.id ||
+        `content_${fileId}_${contentIndex}`;
+      const objectType = rawContent.object || "vector_store.file.content";
       contentItems.push({
         id: contentId,
         object: objectType,
@@ -92,7 +103,7 @@ export class ContentsAPI {
         embedding: embedding,
         metadata: {
           ...chunkMetadata, // chunk_metadata fields from API
-          content_length: content.type === 'text' ? content.text.length : 0,
+          content_length: content.type === "text" ? content.text.length : 0,
         },
       });
     });
@@ -104,7 +115,7 @@ export class ContentsAPI {
     }
 
     return {
-      object: 'list',
+      object: "list",
       data: filteredItems,
       has_more: contentItems.length > (options?.limit || contentItems.length),
     };
diff --git a/llama_stack/ui/lib/format-message-content.test.ts b/llama_stack/ui/lib/format-message-content.test.ts
index cf4055b51..18abbee39 100644
--- a/llama_stack/ui/lib/format-message-content.test.ts
+++ b/llama_stack/ui/lib/format-message-content.test.ts
@@ -18,7 +18,7 @@ describe("extractTextFromContentPart", () => {
   it("should extract text from an array of text content objects", () => {
     const content = [{ type: "text", text: "Which planet do humans live on?" }];
     expect(extractTextFromContentPart(content)).toBe(
-      "Which planet do humans live on?",
+      "Which planet do humans live on?"
     );
   });
 
@@ -37,7 +37,7 @@ describe("extractTextFromContentPart", () => {
       { type: "text", text: "It's an image." },
     ];
     expect(extractTextFromContentPart(content)).toBe(
-      "Look at this: [Image] It's an image.",
+      "Look at this: [Image] It's an image."
     );
   });
 
@@ -53,7 +53,7 @@ describe("extractTextFromContentPart", () => {
   });
 
   it("should handle arrays with plain strings", () => {
-    const content = ["This is", " a test."] as any;
+    const content = ["This is", " a test."] as unknown;
     expect(extractTextFromContentPart(content)).toBe("This is  a test.");
   });
 
@@ -65,7 +65,7 @@ describe("extractTextFromContentPart", () => {
       null,
       undefined,
       { type: "text", noTextProperty: true },
-    ] as any;
+    ] as unknown;
     expect(extractTextFromContentPart(content)).toBe("Valid");
   });
 
@@ -75,15 +75,17 @@ describe("extractTextFromContentPart", () => {
       "Just a string.",
       { type: "image_url", image_url: { url: "http://example.com/image.png" } },
       { type: "text", text: "Last part." },
-    ] as any;
+    ] as unknown;
     expect(extractTextFromContentPart(content)).toBe(
-      "First part. Just a string. [Image] Last part.",
+      "First part. Just a string. [Image] Last part."
     );
   });
 });
 
 describe("extractDisplayableText (composite function)", () => {
-  const mockFormatToolCallToString = (toolCall: any) => {
+  const mockFormatToolCallToString = (toolCall: {
+    function?: { name?: string; arguments?: unknown };
+  }) => {
     if (!toolCall || !toolCall.function || !toolCall.function.name) return "";
     const args = toolCall.function.arguments
       ? JSON.stringify(toolCall.function.arguments)
@@ -125,7 +127,7 @@ describe("extractDisplayableText (composite function)", () => {
       tool_calls: [toolCall],
     };
     expect(extractDisplayableText(messageWithEffectivelyEmptyContent)).toBe(
-      mockFormatToolCallToString(toolCall),
+      mockFormatToolCallToString(toolCall)
     );
 
     const messageWithEmptyContent: ChatMessage = {
@@ -134,7 +136,7 @@ describe("extractDisplayableText (composite function)", () => {
       tool_calls: [toolCall],
     };
     expect(extractDisplayableText(messageWithEmptyContent)).toBe(
-      mockFormatToolCallToString(toolCall),
+      mockFormatToolCallToString(toolCall)
     );
   });
 
@@ -149,7 +151,7 @@ describe("extractDisplayableText (composite function)", () => {
     };
     const expectedToolCallStr = mockFormatToolCallToString(toolCall);
     expect(extractDisplayableText(message)).toBe(
-      `The result is: ${expectedToolCallStr}`,
+      `The result is: ${expectedToolCallStr}`
     );
   });
 
@@ -167,7 +169,7 @@ describe("extractDisplayableText (composite function)", () => {
     };
     const expectedToolCallStr = mockFormatToolCallToString(toolCall);
     expect(extractDisplayableText(message)).toBe(
-      `Okay, checking weather for London. ${expectedToolCallStr}`,
+      `Okay, checking weather for London. ${expectedToolCallStr}`
     );
   });
 
@@ -178,7 +180,7 @@ describe("extractDisplayableText (composite function)", () => {
       tool_calls: [],
     };
     expect(extractDisplayableText(messageEmptyToolCalls)).toBe(
-      "No tools here.",
+      "No tools here."
     );
 
     const messageUndefinedToolCalls: ChatMessage = {
@@ -187,7 +189,7 @@ describe("extractDisplayableText (composite function)", () => {
       tool_calls: undefined,
     };
     expect(extractDisplayableText(messageUndefinedToolCalls)).toBe(
-      "Still no tools.",
+      "Still no tools."
     );
   });
 });
diff --git a/llama_stack/ui/lib/format-message-content.ts b/llama_stack/ui/lib/format-message-content.ts
index 3e7e03a12..ab79775c6 100644
--- a/llama_stack/ui/lib/format-message-content.ts
+++ b/llama_stack/ui/lib/format-message-content.ts
@@ -2,7 +2,7 @@ import { ChatMessage, ChatMessageContentPart } from "@/lib/types";
 import { formatToolCallToString } from "@/lib/format-tool-call";
 
 export function extractTextFromContentPart(
-  content: string | ChatMessageContentPart[] | null | undefined,
+  content: string | ChatMessageContentPart[] | null | undefined
 ): string {
   if (content === null || content === undefined) {
     return "";
@@ -37,7 +37,7 @@ export function extractTextFromContentPart(
 }
 
 export function extractDisplayableText(
-  message: ChatMessage | undefined | null,
+  message: ChatMessage | undefined | null
 ): string {
   if (!message) {
     return "";
diff --git a/llama_stack/ui/lib/format-tool-call.tsx b/llama_stack/ui/lib/format-tool-call.tsx
index f6a286a6e..ec1bdce38 100644
--- a/llama_stack/ui/lib/format-tool-call.tsx
+++ b/llama_stack/ui/lib/format-tool-call.tsx
@@ -5,7 +5,9 @@
  *                 with `name` and `arguments`.
  * @returns A formatted string or an empty string if data is malformed.
  */
-export function formatToolCallToString(toolCall: any): string {
+export function formatToolCallToString(toolCall: {
+  function?: { name?: string; arguments?: unknown };
+}): string {
   if (
     !toolCall ||
     !toolCall.function ||
@@ -24,7 +26,7 @@ export function formatToolCallToString(toolCall: any): string {
   } else {
     try {
       argsString = JSON.stringify(args);
-    } catch (error) {
+    } catch {
       return "";
     }
   }
diff --git a/llama_stack/ui/lib/truncate-text.ts b/llama_stack/ui/lib/truncate-text.ts
index 63e2194f5..59fc1f5ff 100644
--- a/llama_stack/ui/lib/truncate-text.ts
+++ b/llama_stack/ui/lib/truncate-text.ts
@@ -1,6 +1,6 @@
 export function truncateText(
   text: string | null | undefined,
-  maxLength: number = 50,
+  maxLength: number = 50
 ): string {
   if (!text) return "N/A";
   if (text.length <= maxLength) return text;

From 47d5af703c1abcab5112d1806a7abeb00c261413 Mon Sep 17 00:00:00 2001
From: ashwinb <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 00:05:35 +0000
Subject: [PATCH 30/45] chore(responses): Refactor Responses Impl to be
 civilized (#3138)

# What does this PR do?
Refactors the OpenAI responses implementation by extracting streaming and tool execution logic into separate modules. This improves code organization by:

1. Creating a new `StreamingResponseOrchestrator` class in `streaming.py` to handle the streaming response generation logic
2. Moving tool execution functionality to a dedicated `ToolExecutor` class in `tool_executor.py`

## Test Plan

Existing tests
---
 .../k8s-benchmark/openai-mock-server.py       |    0
 .../inline/agents/meta_reference/agents.py    |    2 +-
 .../agents/meta_reference/openai_responses.py | 1154 -----------------
 .../meta_reference/responses/__init__.py      |    5 +
 .../responses/openai_responses.py             |  499 +++++++
 .../meta_reference/responses/streaming.py     |  451 +++++++
 .../meta_reference/responses/tool_executor.py |  365 ++++++
 .../agents/meta_reference/responses/types.py  |   62 +
 .../agents/meta_reference/responses/utils.py  |   50 +
 .../meta_reference/test_openai_responses.py   |    2 +-
 10 files changed, 1434 insertions(+), 1156 deletions(-)
 mode change 100644 => 100755 docs/source/distributions/k8s-benchmark/openai-mock-server.py
 delete mode 100644 llama_stack/providers/inline/agents/meta_reference/openai_responses.py
 create mode 100644 llama_stack/providers/inline/agents/meta_reference/responses/__init__.py
 create mode 100644 llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
 create mode 100644 llama_stack/providers/inline/agents/meta_reference/responses/streaming.py
 create mode 100644 llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py
 create mode 100644 llama_stack/providers/inline/agents/meta_reference/responses/types.py
 create mode 100644 llama_stack/providers/inline/agents/meta_reference/responses/utils.py

diff --git a/docs/source/distributions/k8s-benchmark/openai-mock-server.py b/docs/source/distributions/k8s-benchmark/openai-mock-server.py
old mode 100644
new mode 100755
diff --git a/llama_stack/providers/inline/agents/meta_reference/agents.py b/llama_stack/providers/inline/agents/meta_reference/agents.py
index 0f12a0865..30196c429 100644
--- a/llama_stack/providers/inline/agents/meta_reference/agents.py
+++ b/llama_stack/providers/inline/agents/meta_reference/agents.py
@@ -48,8 +48,8 @@ from llama_stack.providers.utils.responses.responses_store import ResponsesStore
 
 from .agent_instance import ChatAgent
 from .config import MetaReferenceAgentsImplConfig
-from .openai_responses import OpenAIResponsesImpl
 from .persistence import AgentInfo
+from .responses.openai_responses import OpenAIResponsesImpl
 
 logger = logging.getLogger()
 
diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
deleted file mode 100644
index 6aca4d68e..000000000
--- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py
+++ /dev/null
@@ -1,1154 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-import asyncio
-import json
-import time
-import uuid
-from collections.abc import AsyncIterator
-from typing import Any
-
-from openai.types.chat import ChatCompletionToolParam
-from pydantic import BaseModel
-
-from llama_stack.apis.agents import Order
-from llama_stack.apis.agents.openai_responses import (
-    AllowedToolsFilter,
-    ListOpenAIResponseInputItem,
-    ListOpenAIResponseObject,
-    OpenAIDeleteResponseObject,
-    OpenAIResponseContentPartOutputText,
-    OpenAIResponseInput,
-    OpenAIResponseInputFunctionToolCallOutput,
-    OpenAIResponseInputMessageContent,
-    OpenAIResponseInputMessageContentImage,
-    OpenAIResponseInputMessageContentText,
-    OpenAIResponseInputTool,
-    OpenAIResponseInputToolFileSearch,
-    OpenAIResponseInputToolMCP,
-    OpenAIResponseMessage,
-    OpenAIResponseObject,
-    OpenAIResponseObjectStream,
-    OpenAIResponseObjectStreamResponseCompleted,
-    OpenAIResponseObjectStreamResponseContentPartAdded,
-    OpenAIResponseObjectStreamResponseContentPartDone,
-    OpenAIResponseObjectStreamResponseCreated,
-    OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta,
-    OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone,
-    OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta,
-    OpenAIResponseObjectStreamResponseMcpCallArgumentsDone,
-    OpenAIResponseObjectStreamResponseMcpCallCompleted,
-    OpenAIResponseObjectStreamResponseMcpCallFailed,
-    OpenAIResponseObjectStreamResponseMcpCallInProgress,
-    OpenAIResponseObjectStreamResponseOutputItemAdded,
-    OpenAIResponseObjectStreamResponseOutputItemDone,
-    OpenAIResponseObjectStreamResponseOutputTextDelta,
-    OpenAIResponseObjectStreamResponseWebSearchCallCompleted,
-    OpenAIResponseObjectStreamResponseWebSearchCallInProgress,
-    OpenAIResponseObjectStreamResponseWebSearchCallSearching,
-    OpenAIResponseOutput,
-    OpenAIResponseOutputMessageContent,
-    OpenAIResponseOutputMessageContentOutputText,
-    OpenAIResponseOutputMessageFileSearchToolCall,
-    OpenAIResponseOutputMessageFileSearchToolCallResults,
-    OpenAIResponseOutputMessageFunctionToolCall,
-    OpenAIResponseOutputMessageMCPListTools,
-    OpenAIResponseOutputMessageWebSearchToolCall,
-    OpenAIResponseText,
-    OpenAIResponseTextFormat,
-    WebSearchToolTypes,
-)
-from llama_stack.apis.common.content_types import TextContentItem
-from llama_stack.apis.inference import (
-    Inference,
-    OpenAIAssistantMessageParam,
-    OpenAIChatCompletion,
-    OpenAIChatCompletionContentPartImageParam,
-    OpenAIChatCompletionContentPartParam,
-    OpenAIChatCompletionContentPartTextParam,
-    OpenAIChatCompletionToolCall,
-    OpenAIChatCompletionToolCallFunction,
-    OpenAIChoice,
-    OpenAIDeveloperMessageParam,
-    OpenAIImageURL,
-    OpenAIJSONSchema,
-    OpenAIMessageParam,
-    OpenAIResponseFormatJSONObject,
-    OpenAIResponseFormatJSONSchema,
-    OpenAIResponseFormatParam,
-    OpenAIResponseFormatText,
-    OpenAISystemMessageParam,
-    OpenAIToolMessageParam,
-    OpenAIUserMessageParam,
-)
-from llama_stack.apis.tools import ToolGroups, ToolInvocationResult, ToolRuntime
-from llama_stack.apis.vector_io import VectorIO
-from llama_stack.log import get_logger
-from llama_stack.models.llama.datatypes import ToolDefinition, ToolParamDefinition
-from llama_stack.providers.utils.inference.openai_compat import (
-    convert_tooldef_to_openai_tool,
-)
-from llama_stack.providers.utils.responses.responses_store import ResponsesStore
-
-logger = get_logger(name=__name__, category="openai_responses")
-
-OPENAI_RESPONSES_PREFIX = "openai_responses:"
-
-
-class ToolExecutionResult(BaseModel):
-    """Result of streaming tool execution."""
-
-    stream_event: OpenAIResponseObjectStream | None = None
-    sequence_number: int
-    final_output_message: OpenAIResponseOutput | None = None
-    final_input_message: OpenAIMessageParam | None = None
-
-
-async def _convert_response_content_to_chat_content(
-    content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
-) -> str | list[OpenAIChatCompletionContentPartParam]:
-    """
-    Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts.
-
-    The content schemas of each API look similar, but are not exactly the same.
-    """
-    if isinstance(content, str):
-        return content
-
-    converted_parts = []
-    for content_part in content:
-        if isinstance(content_part, OpenAIResponseInputMessageContentText):
-            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
-        elif isinstance(content_part, OpenAIResponseOutputMessageContentOutputText):
-            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
-        elif isinstance(content_part, OpenAIResponseInputMessageContentImage):
-            if content_part.image_url:
-                image_url = OpenAIImageURL(url=content_part.image_url, detail=content_part.detail)
-                converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url))
-        elif isinstance(content_part, str):
-            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part))
-        else:
-            raise ValueError(
-                f"Llama Stack OpenAI Responses does not yet support content type '{type(content_part)}' in this context"
-            )
-    return converted_parts
-
-
-async def _convert_response_input_to_chat_messages(
-    input: str | list[OpenAIResponseInput],
-) -> list[OpenAIMessageParam]:
-    """
-    Convert the input from an OpenAI Response API request into OpenAI Chat Completion messages.
-    """
-    messages: list[OpenAIMessageParam] = []
-    if isinstance(input, list):
-        for input_item in input:
-            if isinstance(input_item, OpenAIResponseInputFunctionToolCallOutput):
-                messages.append(
-                    OpenAIToolMessageParam(
-                        content=input_item.output,
-                        tool_call_id=input_item.call_id,
-                    )
-                )
-            elif isinstance(input_item, OpenAIResponseOutputMessageFunctionToolCall):
-                tool_call = OpenAIChatCompletionToolCall(
-                    index=0,
-                    id=input_item.call_id,
-                    function=OpenAIChatCompletionToolCallFunction(
-                        name=input_item.name,
-                        arguments=input_item.arguments,
-                    ),
-                )
-                messages.append(OpenAIAssistantMessageParam(tool_calls=[tool_call]))
-            else:
-                content = await _convert_response_content_to_chat_content(input_item.content)
-                message_type = await _get_message_type_by_role(input_item.role)
-                if message_type is None:
-                    raise ValueError(
-                        f"Llama Stack OpenAI Responses does not yet support message role '{input_item.role}' in this context"
-                    )
-                messages.append(message_type(content=content))
-    else:
-        messages.append(OpenAIUserMessageParam(content=input))
-    return messages
-
-
-async def _convert_chat_choice_to_response_message(
-    choice: OpenAIChoice,
-) -> OpenAIResponseMessage:
-    """
-    Convert an OpenAI Chat Completion choice into an OpenAI Response output message.
-    """
-    output_content = ""
-    if isinstance(choice.message.content, str):
-        output_content = choice.message.content
-    elif isinstance(choice.message.content, OpenAIChatCompletionContentPartTextParam):
-        output_content = choice.message.content.text
-    else:
-        raise ValueError(
-            f"Llama Stack OpenAI Responses does not yet support output content type: {type(choice.message.content)}"
-        )
-
-    return OpenAIResponseMessage(
-        id=f"msg_{uuid.uuid4()}",
-        content=[OpenAIResponseOutputMessageContentOutputText(text=output_content)],
-        status="completed",
-        role="assistant",
-    )
-
-
-async def _convert_response_text_to_chat_response_format(
-    text: OpenAIResponseText,
-) -> OpenAIResponseFormatParam:
-    """
-    Convert an OpenAI Response text parameter into an OpenAI Chat Completion response format.
-    """
-    if not text.format or text.format["type"] == "text":
-        return OpenAIResponseFormatText(type="text")
-    if text.format["type"] == "json_object":
-        return OpenAIResponseFormatJSONObject()
-    if text.format["type"] == "json_schema":
-        return OpenAIResponseFormatJSONSchema(
-            json_schema=OpenAIJSONSchema(name=text.format["name"], schema=text.format["schema"])
-        )
-    raise ValueError(f"Unsupported text format: {text.format}")
-
-
-async def _get_message_type_by_role(role: str):
-    role_to_type = {
-        "user": OpenAIUserMessageParam,
-        "system": OpenAISystemMessageParam,
-        "assistant": OpenAIAssistantMessageParam,
-        "developer": OpenAIDeveloperMessageParam,
-    }
-    return role_to_type.get(role)
-
-
-class OpenAIResponsePreviousResponseWithInputItems(BaseModel):
-    input_items: ListOpenAIResponseInputItem
-    response: OpenAIResponseObject
-
-
-class ChatCompletionContext(BaseModel):
-    model: str
-    messages: list[OpenAIMessageParam]
-    response_tools: list[OpenAIResponseInputTool] | None = None
-    chat_tools: list[ChatCompletionToolParam] | None = None
-    mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP]
-    temperature: float | None
-    response_format: OpenAIResponseFormatParam
-
-
-class OpenAIResponsesImpl:
-    def __init__(
-        self,
-        inference_api: Inference,
-        tool_groups_api: ToolGroups,
-        tool_runtime_api: ToolRuntime,
-        responses_store: ResponsesStore,
-        vector_io_api: VectorIO,  # VectorIO
-    ):
-        self.inference_api = inference_api
-        self.tool_groups_api = tool_groups_api
-        self.tool_runtime_api = tool_runtime_api
-        self.responses_store = responses_store
-        self.vector_io_api = vector_io_api
-
-    async def _prepend_previous_response(
-        self,
-        input: str | list[OpenAIResponseInput],
-        previous_response_id: str | None = None,
-    ):
-        if previous_response_id:
-            previous_response_with_input = await self.responses_store.get_response_object(previous_response_id)
-
-            # previous response input items
-            new_input_items = previous_response_with_input.input
-
-            # previous response output items
-            new_input_items.extend(previous_response_with_input.output)
-
-            # new input items from the current request
-            if isinstance(input, str):
-                new_input_items.append(OpenAIResponseMessage(content=input, role="user"))
-            else:
-                new_input_items.extend(input)
-
-            input = new_input_items
-
-        return input
-
-    async def _prepend_instructions(self, messages, instructions):
-        if instructions:
-            messages.insert(0, OpenAISystemMessageParam(content=instructions))
-
-    async def get_openai_response(
-        self,
-        response_id: str,
-    ) -> OpenAIResponseObject:
-        response_with_input = await self.responses_store.get_response_object(response_id)
-        return OpenAIResponseObject(**{k: v for k, v in response_with_input.model_dump().items() if k != "input"})
-
-    async def list_openai_responses(
-        self,
-        after: str | None = None,
-        limit: int | None = 50,
-        model: str | None = None,
-        order: Order | None = Order.desc,
-    ) -> ListOpenAIResponseObject:
-        return await self.responses_store.list_responses(after, limit, model, order)
-
-    async def list_openai_response_input_items(
-        self,
-        response_id: str,
-        after: str | None = None,
-        before: str | None = None,
-        include: list[str] | None = None,
-        limit: int | None = 20,
-        order: Order | None = Order.desc,
-    ) -> ListOpenAIResponseInputItem:
-        """List input items for a given OpenAI response.
-
-        :param response_id: The ID of the response to retrieve input items for.
-        :param after: An item ID to list items after, used for pagination.
-        :param before: An item ID to list items before, used for pagination.
-        :param include: Additional fields to include in the response.
-        :param limit: A limit on the number of objects to be returned.
-        :param order: The order to return the input items in.
-        :returns: An ListOpenAIResponseInputItem.
-        """
-        return await self.responses_store.list_response_input_items(response_id, after, before, include, limit, order)
-
-    async def _store_response(
-        self,
-        response: OpenAIResponseObject,
-        input: str | list[OpenAIResponseInput],
-    ) -> None:
-        new_input_id = f"msg_{uuid.uuid4()}"
-        if isinstance(input, str):
-            # synthesize a message from the input string
-            input_content = OpenAIResponseInputMessageContentText(text=input)
-            input_content_item = OpenAIResponseMessage(
-                role="user",
-                content=[input_content],
-                id=new_input_id,
-            )
-            input_items_data = [input_content_item]
-        else:
-            # we already have a list of messages
-            input_items_data = []
-            for input_item in input:
-                if isinstance(input_item, OpenAIResponseMessage):
-                    # These may or may not already have an id, so dump to dict, check for id, and add if missing
-                    input_item_dict = input_item.model_dump()
-                    if "id" not in input_item_dict:
-                        input_item_dict["id"] = new_input_id
-                    input_items_data.append(OpenAIResponseMessage(**input_item_dict))
-                else:
-                    input_items_data.append(input_item)
-
-        await self.responses_store.store_response_object(
-            response_object=response,
-            input=input_items_data,
-        )
-
-    async def create_openai_response(
-        self,
-        input: str | list[OpenAIResponseInput],
-        model: str,
-        instructions: str | None = None,
-        previous_response_id: str | None = None,
-        store: bool | None = True,
-        stream: bool | None = False,
-        temperature: float | None = None,
-        text: OpenAIResponseText | None = None,
-        tools: list[OpenAIResponseInputTool] | None = None,
-        include: list[str] | None = None,
-        max_infer_iters: int | None = 10,
-    ):
-        stream = bool(stream)
-        text = OpenAIResponseText(format=OpenAIResponseTextFormat(type="text")) if text is None else text
-
-        stream_gen = self._create_streaming_response(
-            input=input,
-            model=model,
-            instructions=instructions,
-            previous_response_id=previous_response_id,
-            store=store,
-            temperature=temperature,
-            text=text,
-            tools=tools,
-            max_infer_iters=max_infer_iters,
-        )
-
-        if stream:
-            return stream_gen
-        else:
-            response = None
-            async for stream_chunk in stream_gen:
-                if stream_chunk.type == "response.completed":
-                    if response is not None:
-                        raise ValueError("The response stream completed multiple times! Earlier response: {response}")
-                    response = stream_chunk.response
-                    # don't leave the generator half complete!
-
-            if response is None:
-                raise ValueError("The response stream never completed")
-            return response
-
-    async def _create_streaming_response(
-        self,
-        input: str | list[OpenAIResponseInput],
-        model: str,
-        instructions: str | None = None,
-        previous_response_id: str | None = None,
-        store: bool | None = True,
-        temperature: float | None = None,
-        text: OpenAIResponseText | None = None,
-        tools: list[OpenAIResponseInputTool] | None = None,
-        max_infer_iters: int | None = 10,
-    ) -> AsyncIterator[OpenAIResponseObjectStream]:
-        output_messages: list[OpenAIResponseOutput] = []
-
-        # Input preprocessing
-        input = await self._prepend_previous_response(input, previous_response_id)
-        messages = await _convert_response_input_to_chat_messages(input)
-        await self._prepend_instructions(messages, instructions)
-
-        # Structured outputs
-        response_format = await _convert_response_text_to_chat_response_format(text)
-
-        # Tool setup, TODO: refactor this slightly since this can also yield events
-        chat_tools, mcp_tool_to_server, mcp_list_message = (
-            await self._convert_response_tools_to_chat_tools(tools) if tools else (None, {}, None)
-        )
-        if mcp_list_message:
-            output_messages.append(mcp_list_message)
-
-        ctx = ChatCompletionContext(
-            model=model,
-            messages=messages,
-            response_tools=tools,
-            chat_tools=chat_tools,
-            mcp_tool_to_server=mcp_tool_to_server,
-            temperature=temperature,
-            response_format=response_format,
-        )
-
-        # Create initial response and emit response.created immediately
-        response_id = f"resp-{uuid.uuid4()}"
-        created_at = int(time.time())
-
-        initial_response = OpenAIResponseObject(
-            created_at=created_at,
-            id=response_id,
-            model=model,
-            object="response",
-            status="in_progress",
-            output=output_messages.copy(),
-            text=text,
-        )
-
-        yield OpenAIResponseObjectStreamResponseCreated(response=initial_response)
-
-        n_iter = 0
-        messages = ctx.messages.copy()
-
-        while True:
-            completion_result = await self.inference_api.openai_chat_completion(
-                model=ctx.model,
-                messages=messages,
-                tools=ctx.chat_tools,
-                stream=True,
-                temperature=ctx.temperature,
-                response_format=ctx.response_format,
-            )
-
-            # Process streaming chunks and build complete response
-            chat_response_id = ""
-            chat_response_content = []
-            chat_response_tool_calls: dict[int, OpenAIChatCompletionToolCall] = {}
-            chunk_created = 0
-            chunk_model = ""
-            chunk_finish_reason = ""
-            sequence_number = 0
-
-            # Create a placeholder message item for delta events
-            message_item_id = f"msg_{uuid.uuid4()}"
-            # Track tool call items for streaming events
-            tool_call_item_ids: dict[int, str] = {}
-            # Track content parts for streaming events
-            content_part_emitted = False
-
-            async for chunk in completion_result:
-                chat_response_id = chunk.id
-                chunk_created = chunk.created
-                chunk_model = chunk.model
-                for chunk_choice in chunk.choices:
-                    # Emit incremental text content as delta events
-                    if chunk_choice.delta.content:
-                        # Emit content_part.added event for first text chunk
-                        if not content_part_emitted:
-                            content_part_emitted = True
-                            sequence_number += 1
-                            yield OpenAIResponseObjectStreamResponseContentPartAdded(
-                                response_id=response_id,
-                                item_id=message_item_id,
-                                part=OpenAIResponseContentPartOutputText(
-                                    text="",  # Will be filled incrementally via text deltas
-                                ),
-                                sequence_number=sequence_number,
-                            )
-                        sequence_number += 1
-                        yield OpenAIResponseObjectStreamResponseOutputTextDelta(
-                            content_index=0,
-                            delta=chunk_choice.delta.content,
-                            item_id=message_item_id,
-                            output_index=0,
-                            sequence_number=sequence_number,
-                        )
-
-                    # Collect content for final response
-                    chat_response_content.append(chunk_choice.delta.content or "")
-                    if chunk_choice.finish_reason:
-                        chunk_finish_reason = chunk_choice.finish_reason
-
-                    # Aggregate tool call arguments across chunks
-                    if chunk_choice.delta.tool_calls:
-                        for tool_call in chunk_choice.delta.tool_calls:
-                            response_tool_call = chat_response_tool_calls.get(tool_call.index, None)
-                            # Create new tool call entry if this is the first chunk for this index
-                            is_new_tool_call = response_tool_call is None
-                            if is_new_tool_call:
-                                tool_call_dict: dict[str, Any] = tool_call.model_dump()
-                                tool_call_dict.pop("type", None)
-                                response_tool_call = OpenAIChatCompletionToolCall(**tool_call_dict)
-                                chat_response_tool_calls[tool_call.index] = response_tool_call
-
-                                # Create item ID for this tool call for streaming events
-                                tool_call_item_id = f"fc_{uuid.uuid4()}"
-                                tool_call_item_ids[tool_call.index] = tool_call_item_id
-
-                                # Emit output_item.added event for the new function call
-                                sequence_number += 1
-                                function_call_item = OpenAIResponseOutputMessageFunctionToolCall(
-                                    arguments="",  # Will be filled incrementally via delta events
-                                    call_id=tool_call.id or "",
-                                    name=tool_call.function.name if tool_call.function else "",
-                                    id=tool_call_item_id,
-                                    status="in_progress",
-                                )
-                                yield OpenAIResponseObjectStreamResponseOutputItemAdded(
-                                    response_id=response_id,
-                                    item=function_call_item,
-                                    output_index=len(output_messages),
-                                    sequence_number=sequence_number,
-                                )
-
-                            # Stream tool call arguments as they arrive (differentiate between MCP and function calls)
-                            if tool_call.function and tool_call.function.arguments:
-                                tool_call_item_id = tool_call_item_ids[tool_call.index]
-                                sequence_number += 1
-
-                                # Check if this is an MCP tool call
-                                is_mcp_tool = (
-                                    ctx.mcp_tool_to_server
-                                    and tool_call.function.name
-                                    and tool_call.function.name in ctx.mcp_tool_to_server
-                                )
-                                if is_mcp_tool:
-                                    # Emit MCP-specific argument delta event
-                                    yield OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta(
-                                        delta=tool_call.function.arguments,
-                                        item_id=tool_call_item_id,
-                                        output_index=len(output_messages),
-                                        sequence_number=sequence_number,
-                                    )
-                                else:
-                                    # Emit function call argument delta event
-                                    yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta(
-                                        delta=tool_call.function.arguments,
-                                        item_id=tool_call_item_id,
-                                        output_index=len(output_messages),
-                                        sequence_number=sequence_number,
-                                    )
-
-                                # Accumulate arguments for final response (only for subsequent chunks)
-                                if not is_new_tool_call:
-                                    response_tool_call.function.arguments = (
-                                        response_tool_call.function.arguments or ""
-                                    ) + tool_call.function.arguments
-
-            # Emit arguments.done events for completed tool calls (differentiate between MCP and function calls)
-            for tool_call_index in sorted(chat_response_tool_calls.keys()):
-                tool_call_item_id = tool_call_item_ids[tool_call_index]
-                final_arguments = chat_response_tool_calls[tool_call_index].function.arguments or ""
-                tool_call_name = chat_response_tool_calls[tool_call_index].function.name
-
-                # Check if this is an MCP tool call
-                is_mcp_tool = ctx.mcp_tool_to_server and tool_call_name and tool_call_name in ctx.mcp_tool_to_server
-                sequence_number += 1
-                if is_mcp_tool:
-                    # Emit MCP-specific argument done event
-                    yield OpenAIResponseObjectStreamResponseMcpCallArgumentsDone(
-                        arguments=final_arguments,
-                        item_id=tool_call_item_id,
-                        output_index=len(output_messages),
-                        sequence_number=sequence_number,
-                    )
-                else:
-                    # Emit function call argument done event
-                    yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone(
-                        arguments=final_arguments,
-                        item_id=tool_call_item_id,
-                        output_index=len(output_messages),
-                        sequence_number=sequence_number,
-                    )
-
-            # Convert collected chunks to complete response
-            if chat_response_tool_calls:
-                tool_calls = [chat_response_tool_calls[i] for i in sorted(chat_response_tool_calls.keys())]
-            else:
-                tool_calls = None
-
-            # Emit content_part.done event if text content was streamed (before content gets cleared)
-            if content_part_emitted:
-                final_text = "".join(chat_response_content)
-                sequence_number += 1
-                yield OpenAIResponseObjectStreamResponseContentPartDone(
-                    response_id=response_id,
-                    item_id=message_item_id,
-                    part=OpenAIResponseContentPartOutputText(
-                        text=final_text,
-                    ),
-                    sequence_number=sequence_number,
-                )
-
-            # Clear content when there are tool calls (OpenAI spec behavior)
-            if chat_response_tool_calls:
-                chat_response_content = []
-
-            assistant_message = OpenAIAssistantMessageParam(
-                content="".join(chat_response_content),
-                tool_calls=tool_calls,
-            )
-            current_response = OpenAIChatCompletion(
-                id=chat_response_id,
-                choices=[
-                    OpenAIChoice(
-                        message=assistant_message,
-                        finish_reason=chunk_finish_reason,
-                        index=0,
-                    )
-                ],
-                created=chunk_created,
-                model=chunk_model,
-            )
-
-            function_tool_calls = []
-            non_function_tool_calls = []
-
-            next_turn_messages = messages.copy()
-            for choice in current_response.choices:
-                next_turn_messages.append(choice.message)
-
-                if choice.message.tool_calls and tools:
-                    for tool_call in choice.message.tool_calls:
-                        if _is_function_tool_call(tool_call, tools):
-                            function_tool_calls.append(tool_call)
-                        else:
-                            non_function_tool_calls.append(tool_call)
-                else:
-                    output_messages.append(await _convert_chat_choice_to_response_message(choice))
-
-            # execute non-function tool calls
-            for tool_call in non_function_tool_calls:
-                # Find the item_id for this tool call
-                matching_item_id = None
-                for index, item_id in tool_call_item_ids.items():
-                    response_tool_call = chat_response_tool_calls.get(index)
-                    if response_tool_call and response_tool_call.id == tool_call.id:
-                        matching_item_id = item_id
-                        break
-
-                # Use a fallback item_id if not found
-                if not matching_item_id:
-                    matching_item_id = f"tc_{uuid.uuid4()}"
-
-                # Execute tool call with streaming
-                tool_call_log = None
-                tool_response_message = None
-                async for result in self._execute_tool_call(
-                    tool_call, ctx, sequence_number, response_id, len(output_messages), matching_item_id
-                ):
-                    if result.stream_event:
-                        # Forward streaming events
-                        sequence_number = result.sequence_number
-                        yield result.stream_event
-
-                    if result.final_output_message is not None:
-                        tool_call_log = result.final_output_message
-                        tool_response_message = result.final_input_message
-                        sequence_number = result.sequence_number
-
-                if tool_call_log:
-                    output_messages.append(tool_call_log)
-
-                    # Emit output_item.done event for completed non-function tool call
-                    if matching_item_id:
-                        sequence_number += 1
-                        yield OpenAIResponseObjectStreamResponseOutputItemDone(
-                            response_id=response_id,
-                            item=tool_call_log,
-                            output_index=len(output_messages) - 1,
-                            sequence_number=sequence_number,
-                        )
-
-                if tool_response_message:
-                    next_turn_messages.append(tool_response_message)
-
-            for tool_call in function_tool_calls:
-                # Find the item_id for this tool call from our tracking dictionary
-                matching_item_id = None
-                for index, item_id in tool_call_item_ids.items():
-                    response_tool_call = chat_response_tool_calls.get(index)
-                    if response_tool_call and response_tool_call.id == tool_call.id:
-                        matching_item_id = item_id
-                        break
-
-                # Use existing item_id or create new one if not found
-                final_item_id = matching_item_id or f"fc_{uuid.uuid4()}"
-
-                function_call_item = OpenAIResponseOutputMessageFunctionToolCall(
-                    arguments=tool_call.function.arguments or "",
-                    call_id=tool_call.id,
-                    name=tool_call.function.name or "",
-                    id=final_item_id,
-                    status="completed",
-                )
-                output_messages.append(function_call_item)
-
-                # Emit output_item.done event for completed function call
-                sequence_number += 1
-                yield OpenAIResponseObjectStreamResponseOutputItemDone(
-                    response_id=response_id,
-                    item=function_call_item,
-                    output_index=len(output_messages) - 1,
-                    sequence_number=sequence_number,
-                )
-
-            if not function_tool_calls and not non_function_tool_calls:
-                break
-
-            if function_tool_calls:
-                logger.info("Exiting inference loop since there is a function (client-side) tool call")
-                break
-
-            n_iter += 1
-            if n_iter >= max_infer_iters:
-                logger.info(f"Exiting inference loop since iteration count({n_iter}) exceeds {max_infer_iters=}")
-                break
-
-            messages = next_turn_messages
-
-        # Create final response
-        final_response = OpenAIResponseObject(
-            created_at=created_at,
-            id=response_id,
-            model=model,
-            object="response",
-            status="completed",
-            text=text,
-            output=output_messages,
-        )
-
-        # Emit response.completed
-        yield OpenAIResponseObjectStreamResponseCompleted(response=final_response)
-
-        if store:
-            await self._store_response(
-                response=final_response,
-                input=input,
-            )
-
-    async def delete_openai_response(self, response_id: str) -> OpenAIDeleteResponseObject:
-        return await self.responses_store.delete_response_object(response_id)
-
-    async def _convert_response_tools_to_chat_tools(
-        self, tools: list[OpenAIResponseInputTool]
-    ) -> tuple[
-        list[ChatCompletionToolParam],
-        dict[str, OpenAIResponseInputToolMCP],
-        OpenAIResponseOutput | None,
-    ]:
-        from llama_stack.apis.agents.openai_responses import (
-            MCPListToolsTool,
-        )
-        from llama_stack.apis.tools import Tool
-
-        mcp_tool_to_server = {}
-
-        def make_openai_tool(tool_name: str, tool: Tool) -> ChatCompletionToolParam:
-            tool_def = ToolDefinition(
-                tool_name=tool_name,
-                description=tool.description,
-                parameters={
-                    param.name: ToolParamDefinition(
-                        param_type=param.parameter_type,
-                        description=param.description,
-                        required=param.required,
-                        default=param.default,
-                    )
-                    for param in tool.parameters
-                },
-            )
-            return convert_tooldef_to_openai_tool(tool_def)
-
-        mcp_list_message = None
-        chat_tools: list[ChatCompletionToolParam] = []
-        for input_tool in tools:
-            # TODO: Handle other tool types
-            if input_tool.type == "function":
-                chat_tools.append(ChatCompletionToolParam(type="function", function=input_tool.model_dump()))
-            elif input_tool.type in WebSearchToolTypes:
-                tool_name = "web_search"
-                tool = await self.tool_groups_api.get_tool(tool_name)
-                if not tool:
-                    raise ValueError(f"Tool {tool_name} not found")
-                chat_tools.append(make_openai_tool(tool_name, tool))
-            elif input_tool.type == "file_search":
-                tool_name = "knowledge_search"
-                tool = await self.tool_groups_api.get_tool(tool_name)
-                if not tool:
-                    raise ValueError(f"Tool {tool_name} not found")
-                chat_tools.append(make_openai_tool(tool_name, tool))
-            elif input_tool.type == "mcp":
-                from llama_stack.providers.utils.tools.mcp import list_mcp_tools
-
-                always_allowed = None
-                never_allowed = None
-                if input_tool.allowed_tools:
-                    if isinstance(input_tool.allowed_tools, list):
-                        always_allowed = input_tool.allowed_tools
-                    elif isinstance(input_tool.allowed_tools, AllowedToolsFilter):
-                        always_allowed = input_tool.allowed_tools.always
-                        never_allowed = input_tool.allowed_tools.never
-
-                tool_defs = await list_mcp_tools(
-                    endpoint=input_tool.server_url,
-                    headers=input_tool.headers or {},
-                )
-
-                mcp_list_message = OpenAIResponseOutputMessageMCPListTools(
-                    id=f"mcp_list_{uuid.uuid4()}",
-                    status="completed",
-                    server_label=input_tool.server_label,
-                    tools=[],
-                )
-                for t in tool_defs.data:
-                    if never_allowed and t.name in never_allowed:
-                        continue
-                    if not always_allowed or t.name in always_allowed:
-                        chat_tools.append(make_openai_tool(t.name, t))
-                        if t.name in mcp_tool_to_server:
-                            raise ValueError(f"Duplicate tool name {t.name} found for server {input_tool.server_label}")
-                        mcp_tool_to_server[t.name] = input_tool
-                        mcp_list_message.tools.append(
-                            MCPListToolsTool(
-                                name=t.name,
-                                description=t.description,
-                                input_schema={
-                                    "type": "object",
-                                    "properties": {
-                                        p.name: {
-                                            "type": p.parameter_type,
-                                            "description": p.description,
-                                        }
-                                        for p in t.parameters
-                                    },
-                                    "required": [p.name for p in t.parameters if p.required],
-                                },
-                            )
-                        )
-            else:
-                raise ValueError(f"Llama Stack OpenAI Responses does not yet support tool type: {input_tool.type}")
-        return chat_tools, mcp_tool_to_server, mcp_list_message
-
-    async def _execute_knowledge_search_via_vector_store(
-        self,
-        query: str,
-        response_file_search_tool: OpenAIResponseInputToolFileSearch,
-    ) -> ToolInvocationResult:
-        """Execute knowledge search using vector_stores.search API with filters support."""
-        search_results = []
-
-        # Create search tasks for all vector stores
-        async def search_single_store(vector_store_id):
-            try:
-                search_response = await self.vector_io_api.openai_search_vector_store(
-                    vector_store_id=vector_store_id,
-                    query=query,
-                    filters=response_file_search_tool.filters,
-                    max_num_results=response_file_search_tool.max_num_results,
-                    ranking_options=response_file_search_tool.ranking_options,
-                    rewrite_query=False,
-                )
-                return search_response.data
-            except Exception as e:
-                logger.warning(f"Failed to search vector store {vector_store_id}: {e}")
-                return []
-
-        # Run all searches in parallel using gather
-        search_tasks = [search_single_store(vid) for vid in response_file_search_tool.vector_store_ids]
-        all_results = await asyncio.gather(*search_tasks)
-
-        # Flatten results
-        for results in all_results:
-            search_results.extend(results)
-
-        # Convert search results to tool result format matching memory.py
-        # Format the results as interleaved content similar to memory.py
-        content_items = []
-        content_items.append(
-            TextContentItem(
-                text=f"knowledge_search tool found {len(search_results)} chunks:\nBEGIN of knowledge_search tool results.\n"
-            )
-        )
-
-        for i, result_item in enumerate(search_results):
-            chunk_text = result_item.content[0].text if result_item.content else ""
-            metadata_text = f"document_id: {result_item.file_id}, score: {result_item.score}"
-            if result_item.attributes:
-                metadata_text += f", attributes: {result_item.attributes}"
-            text_content = f"[{i + 1}] {metadata_text}\n{chunk_text}\n"
-            content_items.append(TextContentItem(text=text_content))
-
-        content_items.append(TextContentItem(text="END of knowledge_search tool results.\n"))
-        content_items.append(
-            TextContentItem(
-                text=f'The above results were retrieved to help answer the user\'s query: "{query}". Use them as supporting information only in answering this query.\n',
-            )
-        )
-
-        return ToolInvocationResult(
-            content=content_items,
-            metadata={
-                "document_ids": [r.file_id for r in search_results],
-                "chunks": [r.content[0].text if r.content else "" for r in search_results],
-                "scores": [r.score for r in search_results],
-            },
-        )
-
-    async def _execute_tool_call(
-        self,
-        tool_call: OpenAIChatCompletionToolCall,
-        ctx: ChatCompletionContext,
-        sequence_number: int,
-        response_id: str,
-        output_index: int,
-        item_id: str,
-    ) -> AsyncIterator[ToolExecutionResult]:
-        from llama_stack.providers.utils.inference.prompt_adapter import (
-            interleaved_content_as_str,
-        )
-
-        tool_call_id = tool_call.id
-        function = tool_call.function
-        tool_kwargs = json.loads(function.arguments) if function.arguments else {}
-
-        if not function or not tool_call_id or not function.name:
-            yield ToolExecutionResult(sequence_number=sequence_number)
-            return
-
-        # Emit in_progress event based on tool type (only for tools with specific streaming events)
-        progress_event = None
-        if ctx.mcp_tool_to_server and function.name in ctx.mcp_tool_to_server:
-            sequence_number += 1
-            progress_event = OpenAIResponseObjectStreamResponseMcpCallInProgress(
-                item_id=item_id,
-                output_index=output_index,
-                sequence_number=sequence_number,
-            )
-        elif function.name == "web_search":
-            sequence_number += 1
-            progress_event = OpenAIResponseObjectStreamResponseWebSearchCallInProgress(
-                item_id=item_id,
-                output_index=output_index,
-                sequence_number=sequence_number,
-            )
-        # Note: knowledge_search and other custom tools don't have specific streaming events in OpenAI spec
-
-        if progress_event:
-            yield ToolExecutionResult(stream_event=progress_event, sequence_number=sequence_number)
-
-        # For web search, emit searching event
-        if function.name == "web_search":
-            sequence_number += 1
-            searching_event = OpenAIResponseObjectStreamResponseWebSearchCallSearching(
-                item_id=item_id,
-                output_index=output_index,
-                sequence_number=sequence_number,
-            )
-            yield ToolExecutionResult(stream_event=searching_event, sequence_number=sequence_number)
-
-        # Execute the actual tool call
-        error_exc = None
-        result = None
-        try:
-            if ctx.mcp_tool_to_server and function.name in ctx.mcp_tool_to_server:
-                from llama_stack.providers.utils.tools.mcp import invoke_mcp_tool
-
-                mcp_tool = ctx.mcp_tool_to_server[function.name]
-                result = await invoke_mcp_tool(
-                    endpoint=mcp_tool.server_url,
-                    headers=mcp_tool.headers or {},
-                    tool_name=function.name,
-                    kwargs=tool_kwargs,
-                )
-            elif function.name == "knowledge_search":
-                response_file_search_tool = next(
-                    (t for t in ctx.response_tools if isinstance(t, OpenAIResponseInputToolFileSearch)),
-                    None,
-                )
-                if response_file_search_tool:
-                    # Use vector_stores.search API instead of knowledge_search tool
-                    # to support filters and ranking_options
-                    query = tool_kwargs.get("query", "")
-                    result = await self._execute_knowledge_search_via_vector_store(
-                        query=query,
-                        response_file_search_tool=response_file_search_tool,
-                    )
-            else:
-                result = await self.tool_runtime_api.invoke_tool(
-                    tool_name=function.name,
-                    kwargs=tool_kwargs,
-                )
-        except Exception as e:
-            error_exc = e
-
-        # Emit completion or failure event based on result (only for tools with specific streaming events)
-        has_error = error_exc or (result and ((result.error_code and result.error_code > 0) or result.error_message))
-        completion_event = None
-
-        if ctx.mcp_tool_to_server and function.name in ctx.mcp_tool_to_server:
-            sequence_number += 1
-            if has_error:
-                completion_event = OpenAIResponseObjectStreamResponseMcpCallFailed(
-                    sequence_number=sequence_number,
-                )
-            else:
-                completion_event = OpenAIResponseObjectStreamResponseMcpCallCompleted(
-                    sequence_number=sequence_number,
-                )
-        elif function.name == "web_search":
-            sequence_number += 1
-            completion_event = OpenAIResponseObjectStreamResponseWebSearchCallCompleted(
-                item_id=item_id,
-                output_index=output_index,
-                sequence_number=sequence_number,
-            )
-        # Note: knowledge_search and other custom tools don't have specific completion events in OpenAI spec
-
-        if completion_event:
-            yield ToolExecutionResult(stream_event=completion_event, sequence_number=sequence_number)
-
-        # Build the result message and input message
-        if function.name in ctx.mcp_tool_to_server:
-            from llama_stack.apis.agents.openai_responses import (
-                OpenAIResponseOutputMessageMCPCall,
-            )
-
-            message = OpenAIResponseOutputMessageMCPCall(
-                id=tool_call_id,
-                arguments=function.arguments,
-                name=function.name,
-                server_label=ctx.mcp_tool_to_server[function.name].server_label,
-            )
-            if error_exc:
-                message.error = str(error_exc)
-            elif (result and result.error_code and result.error_code > 0) or (result and result.error_message):
-                message.error = f"Error (code {result.error_code}): {result.error_message}"
-            elif result and result.content:
-                message.output = interleaved_content_as_str(result.content)
-        else:
-            if function.name == "web_search":
-                message = OpenAIResponseOutputMessageWebSearchToolCall(
-                    id=tool_call_id,
-                    status="completed",
-                )
-                if has_error:
-                    message.status = "failed"
-            elif function.name == "knowledge_search":
-                message = OpenAIResponseOutputMessageFileSearchToolCall(
-                    id=tool_call_id,
-                    queries=[tool_kwargs.get("query", "")],
-                    status="completed",
-                )
-                if result and "document_ids" in result.metadata:
-                    message.results = []
-                    for i, doc_id in enumerate(result.metadata["document_ids"]):
-                        text = result.metadata["chunks"][i] if "chunks" in result.metadata else None
-                        score = result.metadata["scores"][i] if "scores" in result.metadata else None
-                        message.results.append(
-                            OpenAIResponseOutputMessageFileSearchToolCallResults(
-                                file_id=doc_id,
-                                filename=doc_id,
-                                text=text,
-                                score=score,
-                                attributes={},
-                            )
-                        )
-                if has_error:
-                    message.status = "failed"
-            else:
-                raise ValueError(f"Unknown tool {function.name} called")
-
-        input_message = None
-        if result and result.content:
-            if isinstance(result.content, str):
-                content = result.content
-            elif isinstance(result.content, list):
-                from llama_stack.apis.common.content_types import (
-                    ImageContentItem,
-                    TextContentItem,
-                )
-
-                content = []
-                for item in result.content:
-                    if isinstance(item, TextContentItem):
-                        part = OpenAIChatCompletionContentPartTextParam(text=item.text)
-                    elif isinstance(item, ImageContentItem):
-                        if item.image.data:
-                            url = f"data:image;base64,{item.image.data}"
-                        else:
-                            url = item.image.url
-                        part = OpenAIChatCompletionContentPartImageParam(image_url=OpenAIImageURL(url=url))
-                    else:
-                        raise ValueError(f"Unknown result content type: {type(item)}")
-                    content.append(part)
-            else:
-                raise ValueError(f"Unknown result content type: {type(result.content)}")
-            input_message = OpenAIToolMessageParam(content=content, tool_call_id=tool_call_id)
-        else:
-            text = str(error_exc) if error_exc else "Tool execution failed"
-            input_message = OpenAIToolMessageParam(content=text, tool_call_id=tool_call_id)
-
-        # Yield the final result
-        yield ToolExecutionResult(
-            sequence_number=sequence_number, final_output_message=message, final_input_message=input_message
-        )
-
-
-def _is_function_tool_call(
-    tool_call: OpenAIChatCompletionToolCall,
-    tools: list[OpenAIResponseInputTool],
-) -> bool:
-    if not tool_call.function:
-        return False
-    for t in tools:
-        if t.type == "function" and t.name == tool_call.function.name:
-            return True
-    return False
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/__init__.py b/llama_stack/providers/inline/agents/meta_reference/responses/__init__.py
new file mode 100644
index 000000000..756f351d8
--- /dev/null
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
new file mode 100644
index 000000000..9a87038bf
--- /dev/null
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
@@ -0,0 +1,499 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import time
+import uuid
+from collections.abc import AsyncIterator
+
+from openai.types.chat import ChatCompletionToolParam
+from pydantic import BaseModel
+
+from llama_stack.apis.agents import Order
+from llama_stack.apis.agents.openai_responses import (
+    AllowedToolsFilter,
+    ListOpenAIResponseInputItem,
+    ListOpenAIResponseObject,
+    MCPListToolsTool,
+    OpenAIDeleteResponseObject,
+    OpenAIResponseInput,
+    OpenAIResponseInputFunctionToolCallOutput,
+    OpenAIResponseInputMessageContent,
+    OpenAIResponseInputMessageContentImage,
+    OpenAIResponseInputMessageContentText,
+    OpenAIResponseInputTool,
+    OpenAIResponseInputToolMCP,
+    OpenAIResponseMessage,
+    OpenAIResponseObject,
+    OpenAIResponseObjectStream,
+    OpenAIResponseOutput,
+    OpenAIResponseOutputMessageContent,
+    OpenAIResponseOutputMessageContentOutputText,
+    OpenAIResponseOutputMessageFunctionToolCall,
+    OpenAIResponseOutputMessageMCPListTools,
+    OpenAIResponseText,
+    OpenAIResponseTextFormat,
+    WebSearchToolTypes,
+)
+from llama_stack.apis.inference import (
+    Inference,
+    OpenAIAssistantMessageParam,
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartParam,
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionToolCall,
+    OpenAIChatCompletionToolCallFunction,
+    OpenAIDeveloperMessageParam,
+    OpenAIImageURL,
+    OpenAIJSONSchema,
+    OpenAIMessageParam,
+    OpenAIResponseFormatJSONObject,
+    OpenAIResponseFormatJSONSchema,
+    OpenAIResponseFormatParam,
+    OpenAIResponseFormatText,
+    OpenAISystemMessageParam,
+    OpenAIToolMessageParam,
+    OpenAIUserMessageParam,
+)
+from llama_stack.apis.tools import Tool, ToolGroups, ToolRuntime
+from llama_stack.apis.vector_io import VectorIO
+from llama_stack.log import get_logger
+from llama_stack.models.llama.datatypes import ToolDefinition, ToolParamDefinition
+from llama_stack.providers.utils.inference.openai_compat import (
+    convert_tooldef_to_openai_tool,
+)
+from llama_stack.providers.utils.responses.responses_store import ResponsesStore
+
+from .streaming import StreamingResponseOrchestrator
+from .tool_executor import ToolExecutor
+from .types import ChatCompletionContext
+
+logger = get_logger(name=__name__, category="responses")
+
+
+async def _convert_response_content_to_chat_content(
+    content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
+) -> str | list[OpenAIChatCompletionContentPartParam]:
+    """
+    Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts.
+
+    The content schemas of each API look similar, but are not exactly the same.
+    """
+    if isinstance(content, str):
+        return content
+
+    converted_parts = []
+    for content_part in content:
+        if isinstance(content_part, OpenAIResponseInputMessageContentText):
+            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
+        elif isinstance(content_part, OpenAIResponseOutputMessageContentOutputText):
+            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
+        elif isinstance(content_part, OpenAIResponseInputMessageContentImage):
+            if content_part.image_url:
+                image_url = OpenAIImageURL(url=content_part.image_url, detail=content_part.detail)
+                converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url))
+        elif isinstance(content_part, str):
+            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part))
+        else:
+            raise ValueError(
+                f"Llama Stack OpenAI Responses does not yet support content type '{type(content_part)}' in this context"
+            )
+    return converted_parts
+
+
+async def _convert_response_input_to_chat_messages(
+    input: str | list[OpenAIResponseInput],
+) -> list[OpenAIMessageParam]:
+    """
+    Convert the input from an OpenAI Response API request into OpenAI Chat Completion messages.
+    """
+    messages: list[OpenAIMessageParam] = []
+    if isinstance(input, list):
+        for input_item in input:
+            if isinstance(input_item, OpenAIResponseInputFunctionToolCallOutput):
+                messages.append(
+                    OpenAIToolMessageParam(
+                        content=input_item.output,
+                        tool_call_id=input_item.call_id,
+                    )
+                )
+            elif isinstance(input_item, OpenAIResponseOutputMessageFunctionToolCall):
+                tool_call = OpenAIChatCompletionToolCall(
+                    index=0,
+                    id=input_item.call_id,
+                    function=OpenAIChatCompletionToolCallFunction(
+                        name=input_item.name,
+                        arguments=input_item.arguments,
+                    ),
+                )
+                messages.append(OpenAIAssistantMessageParam(tool_calls=[tool_call]))
+            else:
+                content = await _convert_response_content_to_chat_content(input_item.content)
+                message_type = await _get_message_type_by_role(input_item.role)
+                if message_type is None:
+                    raise ValueError(
+                        f"Llama Stack OpenAI Responses does not yet support message role '{input_item.role}' in this context"
+                    )
+                messages.append(message_type(content=content))
+    else:
+        messages.append(OpenAIUserMessageParam(content=input))
+    return messages
+
+
+async def _convert_response_text_to_chat_response_format(
+    text: OpenAIResponseText,
+) -> OpenAIResponseFormatParam:
+    """
+    Convert an OpenAI Response text parameter into an OpenAI Chat Completion response format.
+    """
+    if not text.format or text.format["type"] == "text":
+        return OpenAIResponseFormatText(type="text")
+    if text.format["type"] == "json_object":
+        return OpenAIResponseFormatJSONObject()
+    if text.format["type"] == "json_schema":
+        return OpenAIResponseFormatJSONSchema(
+            json_schema=OpenAIJSONSchema(name=text.format["name"], schema=text.format["schema"])
+        )
+    raise ValueError(f"Unsupported text format: {text.format}")
+
+
+async def _get_message_type_by_role(role: str):
+    role_to_type = {
+        "user": OpenAIUserMessageParam,
+        "system": OpenAISystemMessageParam,
+        "assistant": OpenAIAssistantMessageParam,
+        "developer": OpenAIDeveloperMessageParam,
+    }
+    return role_to_type.get(role)
+
+
+class OpenAIResponsePreviousResponseWithInputItems(BaseModel):
+    input_items: ListOpenAIResponseInputItem
+    response: OpenAIResponseObject
+
+
+class OpenAIResponsesImpl:
+    def __init__(
+        self,
+        inference_api: Inference,
+        tool_groups_api: ToolGroups,
+        tool_runtime_api: ToolRuntime,
+        responses_store: ResponsesStore,
+        vector_io_api: VectorIO,  # VectorIO
+    ):
+        self.inference_api = inference_api
+        self.tool_groups_api = tool_groups_api
+        self.tool_runtime_api = tool_runtime_api
+        self.responses_store = responses_store
+        self.vector_io_api = vector_io_api
+        self.tool_executor = ToolExecutor(
+            tool_groups_api=tool_groups_api,
+            tool_runtime_api=tool_runtime_api,
+            vector_io_api=vector_io_api,
+        )
+
+    async def _prepend_previous_response(
+        self,
+        input: str | list[OpenAIResponseInput],
+        previous_response_id: str | None = None,
+    ):
+        if previous_response_id:
+            previous_response_with_input = await self.responses_store.get_response_object(previous_response_id)
+
+            # previous response input items
+            new_input_items = previous_response_with_input.input
+
+            # previous response output items
+            new_input_items.extend(previous_response_with_input.output)
+
+            # new input items from the current request
+            if isinstance(input, str):
+                new_input_items.append(OpenAIResponseMessage(content=input, role="user"))
+            else:
+                new_input_items.extend(input)
+
+            input = new_input_items
+
+        return input
+
+    async def _prepend_instructions(self, messages, instructions):
+        if instructions:
+            messages.insert(0, OpenAISystemMessageParam(content=instructions))
+
+    async def get_openai_response(
+        self,
+        response_id: str,
+    ) -> OpenAIResponseObject:
+        response_with_input = await self.responses_store.get_response_object(response_id)
+        return OpenAIResponseObject(**{k: v for k, v in response_with_input.model_dump().items() if k != "input"})
+
+    async def list_openai_responses(
+        self,
+        after: str | None = None,
+        limit: int | None = 50,
+        model: str | None = None,
+        order: Order | None = Order.desc,
+    ) -> ListOpenAIResponseObject:
+        return await self.responses_store.list_responses(after, limit, model, order)
+
+    async def list_openai_response_input_items(
+        self,
+        response_id: str,
+        after: str | None = None,
+        before: str | None = None,
+        include: list[str] | None = None,
+        limit: int | None = 20,
+        order: Order | None = Order.desc,
+    ) -> ListOpenAIResponseInputItem:
+        """List input items for a given OpenAI response.
+
+        :param response_id: The ID of the response to retrieve input items for.
+        :param after: An item ID to list items after, used for pagination.
+        :param before: An item ID to list items before, used for pagination.
+        :param include: Additional fields to include in the response.
+        :param limit: A limit on the number of objects to be returned.
+        :param order: The order to return the input items in.
+        :returns: An ListOpenAIResponseInputItem.
+        """
+        return await self.responses_store.list_response_input_items(response_id, after, before, include, limit, order)
+
+    async def _store_response(
+        self,
+        response: OpenAIResponseObject,
+        input: str | list[OpenAIResponseInput],
+    ) -> None:
+        new_input_id = f"msg_{uuid.uuid4()}"
+        if isinstance(input, str):
+            # synthesize a message from the input string
+            input_content = OpenAIResponseInputMessageContentText(text=input)
+            input_content_item = OpenAIResponseMessage(
+                role="user",
+                content=[input_content],
+                id=new_input_id,
+            )
+            input_items_data = [input_content_item]
+        else:
+            # we already have a list of messages
+            input_items_data = []
+            for input_item in input:
+                if isinstance(input_item, OpenAIResponseMessage):
+                    # These may or may not already have an id, so dump to dict, check for id, and add if missing
+                    input_item_dict = input_item.model_dump()
+                    if "id" not in input_item_dict:
+                        input_item_dict["id"] = new_input_id
+                    input_items_data.append(OpenAIResponseMessage(**input_item_dict))
+                else:
+                    input_items_data.append(input_item)
+
+        await self.responses_store.store_response_object(
+            response_object=response,
+            input=input_items_data,
+        )
+
+    async def create_openai_response(
+        self,
+        input: str | list[OpenAIResponseInput],
+        model: str,
+        instructions: str | None = None,
+        previous_response_id: str | None = None,
+        store: bool | None = True,
+        stream: bool | None = False,
+        temperature: float | None = None,
+        text: OpenAIResponseText | None = None,
+        tools: list[OpenAIResponseInputTool] | None = None,
+        include: list[str] | None = None,
+        max_infer_iters: int | None = 10,
+    ):
+        stream = bool(stream)
+        text = OpenAIResponseText(format=OpenAIResponseTextFormat(type="text")) if text is None else text
+
+        stream_gen = self._create_streaming_response(
+            input=input,
+            model=model,
+            instructions=instructions,
+            previous_response_id=previous_response_id,
+            store=store,
+            temperature=temperature,
+            text=text,
+            tools=tools,
+            max_infer_iters=max_infer_iters,
+        )
+
+        if stream:
+            return stream_gen
+        else:
+            response = None
+            async for stream_chunk in stream_gen:
+                if stream_chunk.type == "response.completed":
+                    if response is not None:
+                        raise ValueError("The response stream completed multiple times! Earlier response: {response}")
+                    response = stream_chunk.response
+                    # don't leave the generator half complete!
+
+            if response is None:
+                raise ValueError("The response stream never completed")
+            return response
+
+    async def _create_streaming_response(
+        self,
+        input: str | list[OpenAIResponseInput],
+        model: str,
+        instructions: str | None = None,
+        previous_response_id: str | None = None,
+        store: bool | None = True,
+        temperature: float | None = None,
+        text: OpenAIResponseText | None = None,
+        tools: list[OpenAIResponseInputTool] | None = None,
+        max_infer_iters: int | None = 10,
+    ) -> AsyncIterator[OpenAIResponseObjectStream]:
+        # Input preprocessing
+        input = await self._prepend_previous_response(input, previous_response_id)
+        messages = await _convert_response_input_to_chat_messages(input)
+        await self._prepend_instructions(messages, instructions)
+
+        # Structured outputs
+        response_format = await _convert_response_text_to_chat_response_format(text)
+
+        # Tool setup, TODO: refactor this slightly since this can also yield events
+        chat_tools, mcp_tool_to_server, mcp_list_message = (
+            await self._convert_response_tools_to_chat_tools(tools) if tools else (None, {}, None)
+        )
+
+        ctx = ChatCompletionContext(
+            model=model,
+            messages=messages,
+            response_tools=tools,
+            chat_tools=chat_tools,
+            mcp_tool_to_server=mcp_tool_to_server,
+            temperature=temperature,
+            response_format=response_format,
+        )
+
+        # Create orchestrator and delegate streaming logic
+        response_id = f"resp-{uuid.uuid4()}"
+        created_at = int(time.time())
+
+        orchestrator = StreamingResponseOrchestrator(
+            inference_api=self.inference_api,
+            ctx=ctx,
+            response_id=response_id,
+            created_at=created_at,
+            text=text,
+            max_infer_iters=max_infer_iters,
+            tool_executor=self.tool_executor,
+            mcp_list_message=mcp_list_message,
+        )
+
+        # Stream the response
+        final_response = None
+        async for stream_chunk in orchestrator.create_response():
+            if stream_chunk.type == "response.completed":
+                final_response = stream_chunk.response
+            yield stream_chunk
+
+        # Store the response if requested
+        if store and final_response:
+            await self._store_response(
+                response=final_response,
+                input=input,
+            )
+
+    async def delete_openai_response(self, response_id: str) -> OpenAIDeleteResponseObject:
+        return await self.responses_store.delete_response_object(response_id)
+
+    async def _convert_response_tools_to_chat_tools(
+        self, tools: list[OpenAIResponseInputTool]
+    ) -> tuple[
+        list[ChatCompletionToolParam],
+        dict[str, OpenAIResponseInputToolMCP],
+        OpenAIResponseOutput | None,
+    ]:
+        mcp_tool_to_server = {}
+
+        def make_openai_tool(tool_name: str, tool: Tool) -> ChatCompletionToolParam:
+            tool_def = ToolDefinition(
+                tool_name=tool_name,
+                description=tool.description,
+                parameters={
+                    param.name: ToolParamDefinition(
+                        param_type=param.parameter_type,
+                        description=param.description,
+                        required=param.required,
+                        default=param.default,
+                    )
+                    for param in tool.parameters
+                },
+            )
+            return convert_tooldef_to_openai_tool(tool_def)
+
+        mcp_list_message = None
+        chat_tools: list[ChatCompletionToolParam] = []
+        for input_tool in tools:
+            # TODO: Handle other tool types
+            if input_tool.type == "function":
+                chat_tools.append(ChatCompletionToolParam(type="function", function=input_tool.model_dump()))
+            elif input_tool.type in WebSearchToolTypes:
+                tool_name = "web_search"
+                tool = await self.tool_groups_api.get_tool(tool_name)
+                if not tool:
+                    raise ValueError(f"Tool {tool_name} not found")
+                chat_tools.append(make_openai_tool(tool_name, tool))
+            elif input_tool.type == "file_search":
+                tool_name = "knowledge_search"
+                tool = await self.tool_groups_api.get_tool(tool_name)
+                if not tool:
+                    raise ValueError(f"Tool {tool_name} not found")
+                chat_tools.append(make_openai_tool(tool_name, tool))
+            elif input_tool.type == "mcp":
+                from llama_stack.providers.utils.tools.mcp import list_mcp_tools
+
+                always_allowed = None
+                never_allowed = None
+                if input_tool.allowed_tools:
+                    if isinstance(input_tool.allowed_tools, list):
+                        always_allowed = input_tool.allowed_tools
+                    elif isinstance(input_tool.allowed_tools, AllowedToolsFilter):
+                        always_allowed = input_tool.allowed_tools.always
+                        never_allowed = input_tool.allowed_tools.never
+
+                tool_defs = await list_mcp_tools(
+                    endpoint=input_tool.server_url,
+                    headers=input_tool.headers or {},
+                )
+
+                mcp_list_message = OpenAIResponseOutputMessageMCPListTools(
+                    id=f"mcp_list_{uuid.uuid4()}",
+                    status="completed",
+                    server_label=input_tool.server_label,
+                    tools=[],
+                )
+                for t in tool_defs.data:
+                    if never_allowed and t.name in never_allowed:
+                        continue
+                    if not always_allowed or t.name in always_allowed:
+                        chat_tools.append(make_openai_tool(t.name, t))
+                        if t.name in mcp_tool_to_server:
+                            raise ValueError(f"Duplicate tool name {t.name} found for server {input_tool.server_label}")
+                        mcp_tool_to_server[t.name] = input_tool
+                        mcp_list_message.tools.append(
+                            MCPListToolsTool(
+                                name=t.name,
+                                description=t.description,
+                                input_schema={
+                                    "type": "object",
+                                    "properties": {
+                                        p.name: {
+                                            "type": p.parameter_type,
+                                            "description": p.description,
+                                        }
+                                        for p in t.parameters
+                                    },
+                                    "required": [p.name for p in t.parameters if p.required],
+                                },
+                            )
+                        )
+            else:
+                raise ValueError(f"Llama Stack OpenAI Responses does not yet support tool type: {input_tool.type}")
+        return chat_tools, mcp_tool_to_server, mcp_list_message
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py b/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py
new file mode 100644
index 000000000..2e4ce0c37
--- /dev/null
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py
@@ -0,0 +1,451 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import uuid
+from collections.abc import AsyncIterator
+from typing import Any
+
+from llama_stack.apis.agents.openai_responses import (
+    OpenAIResponseContentPartOutputText,
+    OpenAIResponseObject,
+    OpenAIResponseObjectStream,
+    OpenAIResponseObjectStreamResponseCompleted,
+    OpenAIResponseObjectStreamResponseContentPartAdded,
+    OpenAIResponseObjectStreamResponseContentPartDone,
+    OpenAIResponseObjectStreamResponseCreated,
+    OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta,
+    OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone,
+    OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta,
+    OpenAIResponseObjectStreamResponseMcpCallArgumentsDone,
+    OpenAIResponseObjectStreamResponseOutputItemAdded,
+    OpenAIResponseObjectStreamResponseOutputItemDone,
+    OpenAIResponseObjectStreamResponseOutputTextDelta,
+    OpenAIResponseOutput,
+    OpenAIResponseOutputMessageFunctionToolCall,
+    OpenAIResponseText,
+)
+from llama_stack.apis.inference import (
+    Inference,
+    OpenAIAssistantMessageParam,
+    OpenAIChatCompletion,
+    OpenAIChatCompletionToolCall,
+    OpenAIChoice,
+)
+from llama_stack.log import get_logger
+
+from .types import ChatCompletionContext, ChatCompletionResult
+from .utils import convert_chat_choice_to_response_message, is_function_tool_call
+
+logger = get_logger(name=__name__, category="responses")
+
+
+class StreamingResponseOrchestrator:
+    def __init__(
+        self,
+        inference_api: Inference,
+        ctx: ChatCompletionContext,
+        response_id: str,
+        created_at: int,
+        text: OpenAIResponseText,
+        max_infer_iters: int,
+        tool_executor,  # Will be the tool execution logic from the main class
+        mcp_list_message: OpenAIResponseOutput | None = None,
+    ):
+        self.inference_api = inference_api
+        self.ctx = ctx
+        self.response_id = response_id
+        self.created_at = created_at
+        self.text = text
+        self.max_infer_iters = max_infer_iters
+        self.tool_executor = tool_executor
+        self.sequence_number = 0
+        self.mcp_list_message = mcp_list_message
+
+    async def create_response(self) -> AsyncIterator[OpenAIResponseObjectStream]:
+        # Initialize output messages with MCP list message if present
+        output_messages: list[OpenAIResponseOutput] = []
+        if self.mcp_list_message:
+            output_messages.append(self.mcp_list_message)
+        # Create initial response and emit response.created immediately
+        initial_response = OpenAIResponseObject(
+            created_at=self.created_at,
+            id=self.response_id,
+            model=self.ctx.model,
+            object="response",
+            status="in_progress",
+            output=output_messages.copy(),
+            text=self.text,
+        )
+
+        yield OpenAIResponseObjectStreamResponseCreated(response=initial_response)
+
+        n_iter = 0
+        messages = self.ctx.messages.copy()
+
+        while True:
+            completion_result = await self.inference_api.openai_chat_completion(
+                model=self.ctx.model,
+                messages=messages,
+                tools=self.ctx.chat_tools,
+                stream=True,
+                temperature=self.ctx.temperature,
+                response_format=self.ctx.response_format,
+            )
+
+            # Process streaming chunks and build complete response
+            completion_result_data = None
+            async for stream_event_or_result in self._process_streaming_chunks(completion_result, output_messages):
+                if isinstance(stream_event_or_result, ChatCompletionResult):
+                    completion_result_data = stream_event_or_result
+                else:
+                    yield stream_event_or_result
+            if not completion_result_data:
+                raise ValueError("Streaming chunk processor failed to return completion data")
+            current_response = self._build_chat_completion(completion_result_data)
+
+            function_tool_calls, non_function_tool_calls, next_turn_messages = self._separate_tool_calls(
+                current_response, messages
+            )
+
+            # Handle choices with no tool calls
+            for choice in current_response.choices:
+                if not (choice.message.tool_calls and self.ctx.response_tools):
+                    output_messages.append(await convert_chat_choice_to_response_message(choice))
+
+            # Execute tool calls and coordinate results
+            async for stream_event in self._coordinate_tool_execution(
+                function_tool_calls,
+                non_function_tool_calls,
+                completion_result_data,
+                output_messages,
+                next_turn_messages,
+            ):
+                yield stream_event
+
+            if not function_tool_calls and not non_function_tool_calls:
+                break
+
+            if function_tool_calls:
+                logger.info("Exiting inference loop since there is a function (client-side) tool call")
+                break
+
+            n_iter += 1
+            if n_iter >= self.max_infer_iters:
+                logger.info(f"Exiting inference loop since iteration count({n_iter}) exceeds {self.max_infer_iters=}")
+                break
+
+            messages = next_turn_messages
+
+        # Create final response
+        final_response = OpenAIResponseObject(
+            created_at=self.created_at,
+            id=self.response_id,
+            model=self.ctx.model,
+            object="response",
+            status="completed",
+            text=self.text,
+            output=output_messages,
+        )
+
+        # Emit response.completed
+        yield OpenAIResponseObjectStreamResponseCompleted(response=final_response)
+
+    def _separate_tool_calls(self, current_response, messages) -> tuple[list, list, list]:
+        """Separate tool calls into function and non-function categories."""
+        function_tool_calls = []
+        non_function_tool_calls = []
+        next_turn_messages = messages.copy()
+
+        for choice in current_response.choices:
+            next_turn_messages.append(choice.message)
+
+            if choice.message.tool_calls and self.ctx.response_tools:
+                for tool_call in choice.message.tool_calls:
+                    if is_function_tool_call(tool_call, self.ctx.response_tools):
+                        function_tool_calls.append(tool_call)
+                    else:
+                        non_function_tool_calls.append(tool_call)
+
+        return function_tool_calls, non_function_tool_calls, next_turn_messages
+
+    async def _process_streaming_chunks(
+        self, completion_result, output_messages: list[OpenAIResponseOutput]
+    ) -> AsyncIterator[OpenAIResponseObjectStream | ChatCompletionResult]:
+        """Process streaming chunks and emit events, returning completion data."""
+        # Initialize result tracking
+        chat_response_id = ""
+        chat_response_content = []
+        chat_response_tool_calls: dict[int, OpenAIChatCompletionToolCall] = {}
+        chunk_created = 0
+        chunk_model = ""
+        chunk_finish_reason = ""
+
+        # Create a placeholder message item for delta events
+        message_item_id = f"msg_{uuid.uuid4()}"
+        # Track tool call items for streaming events
+        tool_call_item_ids: dict[int, str] = {}
+        # Track content parts for streaming events
+        content_part_emitted = False
+
+        async for chunk in completion_result:
+            chat_response_id = chunk.id
+            chunk_created = chunk.created
+            chunk_model = chunk.model
+            for chunk_choice in chunk.choices:
+                # Emit incremental text content as delta events
+                if chunk_choice.delta.content:
+                    # Emit content_part.added event for first text chunk
+                    if not content_part_emitted:
+                        content_part_emitted = True
+                        self.sequence_number += 1
+                        yield OpenAIResponseObjectStreamResponseContentPartAdded(
+                            response_id=self.response_id,
+                            item_id=message_item_id,
+                            part=OpenAIResponseContentPartOutputText(
+                                text="",  # Will be filled incrementally via text deltas
+                            ),
+                            sequence_number=self.sequence_number,
+                        )
+                    self.sequence_number += 1
+                    yield OpenAIResponseObjectStreamResponseOutputTextDelta(
+                        content_index=0,
+                        delta=chunk_choice.delta.content,
+                        item_id=message_item_id,
+                        output_index=0,
+                        sequence_number=self.sequence_number,
+                    )
+
+                # Collect content for final response
+                chat_response_content.append(chunk_choice.delta.content or "")
+                if chunk_choice.finish_reason:
+                    chunk_finish_reason = chunk_choice.finish_reason
+
+                # Aggregate tool call arguments across chunks
+                if chunk_choice.delta.tool_calls:
+                    for tool_call in chunk_choice.delta.tool_calls:
+                        response_tool_call = chat_response_tool_calls.get(tool_call.index, None)
+                        # Create new tool call entry if this is the first chunk for this index
+                        is_new_tool_call = response_tool_call is None
+                        if is_new_tool_call:
+                            tool_call_dict: dict[str, Any] = tool_call.model_dump()
+                            tool_call_dict.pop("type", None)
+                            response_tool_call = OpenAIChatCompletionToolCall(**tool_call_dict)
+                            chat_response_tool_calls[tool_call.index] = response_tool_call
+
+                            # Create item ID for this tool call for streaming events
+                            tool_call_item_id = f"fc_{uuid.uuid4()}"
+                            tool_call_item_ids[tool_call.index] = tool_call_item_id
+
+                            # Emit output_item.added event for the new function call
+                            self.sequence_number += 1
+                            function_call_item = OpenAIResponseOutputMessageFunctionToolCall(
+                                arguments="",  # Will be filled incrementally via delta events
+                                call_id=tool_call.id or "",
+                                name=tool_call.function.name if tool_call.function else "",
+                                id=tool_call_item_id,
+                                status="in_progress",
+                            )
+                            yield OpenAIResponseObjectStreamResponseOutputItemAdded(
+                                response_id=self.response_id,
+                                item=function_call_item,
+                                output_index=len(output_messages),
+                                sequence_number=self.sequence_number,
+                            )
+
+                        # Stream tool call arguments as they arrive (differentiate between MCP and function calls)
+                        if tool_call.function and tool_call.function.arguments:
+                            tool_call_item_id = tool_call_item_ids[tool_call.index]
+                            self.sequence_number += 1
+
+                            # Check if this is an MCP tool call
+                            is_mcp_tool = (
+                                tool_call.function.name and tool_call.function.name in self.ctx.mcp_tool_to_server
+                            )
+                            if is_mcp_tool:
+                                # Emit MCP-specific argument delta event
+                                yield OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta(
+                                    delta=tool_call.function.arguments,
+                                    item_id=tool_call_item_id,
+                                    output_index=len(output_messages),
+                                    sequence_number=self.sequence_number,
+                                )
+                            else:
+                                # Emit function call argument delta event
+                                yield OpenAIResponseObjectStreamResponseFunctionCallArgumentsDelta(
+                                    delta=tool_call.function.arguments,
+                                    item_id=tool_call_item_id,
+                                    output_index=len(output_messages),
+                                    sequence_number=self.sequence_number,
+                                )
+
+                            # Accumulate arguments for final response (only for subsequent chunks)
+                            if not is_new_tool_call:
+                                response_tool_call.function.arguments = (
+                                    response_tool_call.function.arguments or ""
+                                ) + tool_call.function.arguments
+
+        # Emit arguments.done events for completed tool calls (differentiate between MCP and function calls)
+        for tool_call_index in sorted(chat_response_tool_calls.keys()):
+            tool_call_item_id = tool_call_item_ids[tool_call_index]
+            final_arguments = chat_response_tool_calls[tool_call_index].function.arguments or ""
+            tool_call_name = chat_response_tool_calls[tool_call_index].function.name
+
+            # Check if this is an MCP tool call
+            is_mcp_tool = (
+                self.ctx.mcp_tool_to_server and tool_call_name and tool_call_name in self.ctx.mcp_tool_to_server
+            )
+            self.sequence_number += 1
+            done_event_cls = (
+                OpenAIResponseObjectStreamResponseMcpCallArgumentsDone
+                if is_mcp_tool
+                else OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone
+            )
+            yield done_event_cls(
+                arguments=final_arguments,
+                item_id=tool_call_item_id,
+                output_index=len(output_messages),
+                sequence_number=self.sequence_number,
+            )
+
+        # Emit content_part.done event if text content was streamed (before content gets cleared)
+        if content_part_emitted:
+            final_text = "".join(chat_response_content)
+            self.sequence_number += 1
+            yield OpenAIResponseObjectStreamResponseContentPartDone(
+                response_id=self.response_id,
+                item_id=message_item_id,
+                part=OpenAIResponseContentPartOutputText(
+                    text=final_text,
+                ),
+                sequence_number=self.sequence_number,
+            )
+
+        # Clear content when there are tool calls (OpenAI spec behavior)
+        if chat_response_tool_calls:
+            chat_response_content = []
+
+        yield ChatCompletionResult(
+            response_id=chat_response_id,
+            content=chat_response_content,
+            tool_calls=chat_response_tool_calls,
+            created=chunk_created,
+            model=chunk_model,
+            finish_reason=chunk_finish_reason,
+            message_item_id=message_item_id,
+            tool_call_item_ids=tool_call_item_ids,
+            content_part_emitted=content_part_emitted,
+        )
+
+    def _build_chat_completion(self, result: ChatCompletionResult) -> OpenAIChatCompletion:
+        """Build OpenAIChatCompletion from ChatCompletionResult."""
+        # Convert collected chunks to complete response
+        if result.tool_calls:
+            tool_calls = [result.tool_calls[i] for i in sorted(result.tool_calls.keys())]
+        else:
+            tool_calls = None
+
+        assistant_message = OpenAIAssistantMessageParam(
+            content=result.content_text,
+            tool_calls=tool_calls,
+        )
+        return OpenAIChatCompletion(
+            id=result.response_id,
+            choices=[
+                OpenAIChoice(
+                    message=assistant_message,
+                    finish_reason=result.finish_reason,
+                    index=0,
+                )
+            ],
+            created=result.created,
+            model=result.model,
+        )
+
+    async def _coordinate_tool_execution(
+        self,
+        function_tool_calls: list,
+        non_function_tool_calls: list,
+        completion_result_data: ChatCompletionResult,
+        output_messages: list[OpenAIResponseOutput],
+        next_turn_messages: list,
+    ) -> AsyncIterator[OpenAIResponseObjectStream]:
+        """Coordinate execution of both function and non-function tool calls."""
+        # Execute non-function tool calls
+        for tool_call in non_function_tool_calls:
+            # Find the item_id for this tool call
+            matching_item_id = None
+            for index, item_id in completion_result_data.tool_call_item_ids.items():
+                response_tool_call = completion_result_data.tool_calls.get(index)
+                if response_tool_call and response_tool_call.id == tool_call.id:
+                    matching_item_id = item_id
+                    break
+
+            # Use a fallback item_id if not found
+            if not matching_item_id:
+                matching_item_id = f"tc_{uuid.uuid4()}"
+
+            # Execute tool call with streaming
+            tool_call_log = None
+            tool_response_message = None
+            async for result in self.tool_executor.execute_tool_call(
+                tool_call, self.ctx, self.sequence_number, len(output_messages), matching_item_id
+            ):
+                if result.stream_event:
+                    # Forward streaming events
+                    self.sequence_number = result.sequence_number
+                    yield result.stream_event
+
+                if result.final_output_message is not None:
+                    tool_call_log = result.final_output_message
+                    tool_response_message = result.final_input_message
+                    self.sequence_number = result.sequence_number
+
+            if tool_call_log:
+                output_messages.append(tool_call_log)
+
+                # Emit output_item.done event for completed non-function tool call
+                if matching_item_id:
+                    self.sequence_number += 1
+                    yield OpenAIResponseObjectStreamResponseOutputItemDone(
+                        response_id=self.response_id,
+                        item=tool_call_log,
+                        output_index=len(output_messages) - 1,
+                        sequence_number=self.sequence_number,
+                    )
+
+            if tool_response_message:
+                next_turn_messages.append(tool_response_message)
+
+        # Execute function tool calls (client-side)
+        for tool_call in function_tool_calls:
+            # Find the item_id for this tool call from our tracking dictionary
+            matching_item_id = None
+            for index, item_id in completion_result_data.tool_call_item_ids.items():
+                response_tool_call = completion_result_data.tool_calls.get(index)
+                if response_tool_call and response_tool_call.id == tool_call.id:
+                    matching_item_id = item_id
+                    break
+
+            # Use existing item_id or create new one if not found
+            final_item_id = matching_item_id or f"fc_{uuid.uuid4()}"
+
+            function_call_item = OpenAIResponseOutputMessageFunctionToolCall(
+                arguments=tool_call.function.arguments or "",
+                call_id=tool_call.id,
+                name=tool_call.function.name or "",
+                id=final_item_id,
+                status="completed",
+            )
+            output_messages.append(function_call_item)
+
+            # Emit output_item.done event for completed function call
+            self.sequence_number += 1
+            yield OpenAIResponseObjectStreamResponseOutputItemDone(
+                response_id=self.response_id,
+                item=function_call_item,
+                output_index=len(output_messages) - 1,
+                sequence_number=self.sequence_number,
+            )
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py b/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py
new file mode 100644
index 000000000..6b7845138
--- /dev/null
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py
@@ -0,0 +1,365 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import asyncio
+import json
+from collections.abc import AsyncIterator
+
+from llama_stack.apis.agents.openai_responses import (
+    OpenAIResponseInputToolFileSearch,
+    OpenAIResponseObjectStreamResponseMcpCallCompleted,
+    OpenAIResponseObjectStreamResponseMcpCallFailed,
+    OpenAIResponseObjectStreamResponseMcpCallInProgress,
+    OpenAIResponseObjectStreamResponseWebSearchCallCompleted,
+    OpenAIResponseObjectStreamResponseWebSearchCallInProgress,
+    OpenAIResponseObjectStreamResponseWebSearchCallSearching,
+    OpenAIResponseOutputMessageFileSearchToolCall,
+    OpenAIResponseOutputMessageFileSearchToolCallResults,
+    OpenAIResponseOutputMessageWebSearchToolCall,
+)
+from llama_stack.apis.common.content_types import (
+    ImageContentItem,
+    TextContentItem,
+)
+from llama_stack.apis.inference import (
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionToolCall,
+    OpenAIImageURL,
+    OpenAIToolMessageParam,
+)
+from llama_stack.apis.tools import ToolGroups, ToolInvocationResult, ToolRuntime
+from llama_stack.apis.vector_io import VectorIO
+from llama_stack.log import get_logger
+
+from .types import ChatCompletionContext, ToolExecutionResult
+
+logger = get_logger(name=__name__, category="responses")
+
+
+class ToolExecutor:
+    def __init__(
+        self,
+        tool_groups_api: ToolGroups,
+        tool_runtime_api: ToolRuntime,
+        vector_io_api: VectorIO,
+    ):
+        self.tool_groups_api = tool_groups_api
+        self.tool_runtime_api = tool_runtime_api
+        self.vector_io_api = vector_io_api
+
+    async def execute_tool_call(
+        self,
+        tool_call: OpenAIChatCompletionToolCall,
+        ctx: ChatCompletionContext,
+        sequence_number: int,
+        output_index: int,
+        item_id: str,
+    ) -> AsyncIterator[ToolExecutionResult]:
+        tool_call_id = tool_call.id
+        function = tool_call.function
+        tool_kwargs = json.loads(function.arguments) if function.arguments else {}
+
+        if not function or not tool_call_id or not function.name:
+            yield ToolExecutionResult(sequence_number=sequence_number)
+            return
+
+        # Emit progress events for tool execution start
+        async for event_result in self._emit_progress_events(
+            function.name, ctx, sequence_number, output_index, item_id
+        ):
+            sequence_number = event_result.sequence_number
+            yield event_result
+
+        # Execute the actual tool call
+        error_exc, result = await self._execute_tool(function.name, tool_kwargs, ctx)
+
+        # Emit completion events for tool execution
+        has_error = error_exc or (result and ((result.error_code and result.error_code > 0) or result.error_message))
+        async for event_result in self._emit_completion_events(
+            function.name, ctx, sequence_number, output_index, item_id, has_error
+        ):
+            sequence_number = event_result.sequence_number
+            yield event_result
+
+        # Build result messages from tool execution
+        output_message, input_message = await self._build_result_messages(
+            function, tool_call_id, tool_kwargs, ctx, error_exc, result, has_error
+        )
+
+        # Yield the final result
+        yield ToolExecutionResult(
+            sequence_number=sequence_number, final_output_message=output_message, final_input_message=input_message
+        )
+
+    async def _execute_knowledge_search_via_vector_store(
+        self,
+        query: str,
+        response_file_search_tool: OpenAIResponseInputToolFileSearch,
+    ) -> ToolInvocationResult:
+        """Execute knowledge search using vector_stores.search API with filters support."""
+        search_results = []
+
+        # Create search tasks for all vector stores
+        async def search_single_store(vector_store_id):
+            try:
+                search_response = await self.vector_io_api.openai_search_vector_store(
+                    vector_store_id=vector_store_id,
+                    query=query,
+                    filters=response_file_search_tool.filters,
+                    max_num_results=response_file_search_tool.max_num_results,
+                    ranking_options=response_file_search_tool.ranking_options,
+                    rewrite_query=False,
+                )
+                return search_response.data
+            except Exception as e:
+                logger.warning(f"Failed to search vector store {vector_store_id}: {e}")
+                return []
+
+        # Run all searches in parallel using gather
+        search_tasks = [search_single_store(vid) for vid in response_file_search_tool.vector_store_ids]
+        all_results = await asyncio.gather(*search_tasks)
+
+        # Flatten results
+        for results in all_results:
+            search_results.extend(results)
+
+        # Convert search results to tool result format matching memory.py
+        # Format the results as interleaved content similar to memory.py
+        content_items = []
+        content_items.append(
+            TextContentItem(
+                text=f"knowledge_search tool found {len(search_results)} chunks:\nBEGIN of knowledge_search tool results.\n"
+            )
+        )
+
+        for i, result_item in enumerate(search_results):
+            chunk_text = result_item.content[0].text if result_item.content else ""
+            metadata_text = f"document_id: {result_item.file_id}, score: {result_item.score}"
+            if result_item.attributes:
+                metadata_text += f", attributes: {result_item.attributes}"
+            text_content = f"[{i + 1}] {metadata_text}\n{chunk_text}\n"
+            content_items.append(TextContentItem(text=text_content))
+
+        content_items.append(TextContentItem(text="END of knowledge_search tool results.\n"))
+        content_items.append(
+            TextContentItem(
+                text=f'The above results were retrieved to help answer the user\'s query: "{query}". Use them as supporting information only in answering this query.\n',
+            )
+        )
+
+        return ToolInvocationResult(
+            content=content_items,
+            metadata={
+                "document_ids": [r.file_id for r in search_results],
+                "chunks": [r.content[0].text if r.content else "" for r in search_results],
+                "scores": [r.score for r in search_results],
+            },
+        )
+
+    async def _emit_progress_events(
+        self, function_name: str, ctx: ChatCompletionContext, sequence_number: int, output_index: int, item_id: str
+    ) -> AsyncIterator[ToolExecutionResult]:
+        """Emit progress events for tool execution start."""
+        # Emit in_progress event based on tool type (only for tools with specific streaming events)
+        progress_event = None
+        if ctx.mcp_tool_to_server and function_name in ctx.mcp_tool_to_server:
+            sequence_number += 1
+            progress_event = OpenAIResponseObjectStreamResponseMcpCallInProgress(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+        elif function_name == "web_search":
+            sequence_number += 1
+            progress_event = OpenAIResponseObjectStreamResponseWebSearchCallInProgress(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+        # Note: knowledge_search and other custom tools don't have specific streaming events in OpenAI spec
+
+        if progress_event:
+            yield ToolExecutionResult(stream_event=progress_event, sequence_number=sequence_number)
+
+        # For web search, emit searching event
+        if function_name == "web_search":
+            sequence_number += 1
+            searching_event = OpenAIResponseObjectStreamResponseWebSearchCallSearching(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+            yield ToolExecutionResult(stream_event=searching_event, sequence_number=sequence_number)
+
+    async def _execute_tool(
+        self, function_name: str, tool_kwargs: dict, ctx: ChatCompletionContext
+    ) -> tuple[Exception | None, any]:
+        """Execute the tool and return error exception and result."""
+        error_exc = None
+        result = None
+
+        try:
+            if ctx.mcp_tool_to_server and function_name in ctx.mcp_tool_to_server:
+                from llama_stack.providers.utils.tools.mcp import invoke_mcp_tool
+
+                mcp_tool = ctx.mcp_tool_to_server[function_name]
+                result = await invoke_mcp_tool(
+                    endpoint=mcp_tool.server_url,
+                    headers=mcp_tool.headers or {},
+                    tool_name=function_name,
+                    kwargs=tool_kwargs,
+                )
+            elif function_name == "knowledge_search":
+                response_file_search_tool = next(
+                    (t for t in ctx.response_tools if isinstance(t, OpenAIResponseInputToolFileSearch)),
+                    None,
+                )
+                if response_file_search_tool:
+                    # Use vector_stores.search API instead of knowledge_search tool
+                    # to support filters and ranking_options
+                    query = tool_kwargs.get("query", "")
+                    result = await self._execute_knowledge_search_via_vector_store(
+                        query=query,
+                        response_file_search_tool=response_file_search_tool,
+                    )
+            else:
+                result = await self.tool_runtime_api.invoke_tool(
+                    tool_name=function_name,
+                    kwargs=tool_kwargs,
+                )
+        except Exception as e:
+            error_exc = e
+
+        return error_exc, result
+
+    async def _emit_completion_events(
+        self,
+        function_name: str,
+        ctx: ChatCompletionContext,
+        sequence_number: int,
+        output_index: int,
+        item_id: str,
+        has_error: bool,
+    ) -> AsyncIterator[ToolExecutionResult]:
+        """Emit completion or failure events for tool execution."""
+        completion_event = None
+
+        if ctx.mcp_tool_to_server and function_name in ctx.mcp_tool_to_server:
+            sequence_number += 1
+            if has_error:
+                completion_event = OpenAIResponseObjectStreamResponseMcpCallFailed(
+                    sequence_number=sequence_number,
+                )
+            else:
+                completion_event = OpenAIResponseObjectStreamResponseMcpCallCompleted(
+                    sequence_number=sequence_number,
+                )
+        elif function_name == "web_search":
+            sequence_number += 1
+            completion_event = OpenAIResponseObjectStreamResponseWebSearchCallCompleted(
+                item_id=item_id,
+                output_index=output_index,
+                sequence_number=sequence_number,
+            )
+        # Note: knowledge_search and other custom tools don't have specific completion events in OpenAI spec
+
+        if completion_event:
+            yield ToolExecutionResult(stream_event=completion_event, sequence_number=sequence_number)
+
+    async def _build_result_messages(
+        self,
+        function,
+        tool_call_id: str,
+        tool_kwargs: dict,
+        ctx: ChatCompletionContext,
+        error_exc: Exception | None,
+        result: any,
+        has_error: bool,
+    ) -> tuple[any, any]:
+        """Build output and input messages from tool execution results."""
+        from llama_stack.providers.utils.inference.prompt_adapter import (
+            interleaved_content_as_str,
+        )
+
+        # Build output message
+        if function.name in ctx.mcp_tool_to_server:
+            from llama_stack.apis.agents.openai_responses import (
+                OpenAIResponseOutputMessageMCPCall,
+            )
+
+            message = OpenAIResponseOutputMessageMCPCall(
+                id=tool_call_id,
+                arguments=function.arguments,
+                name=function.name,
+                server_label=ctx.mcp_tool_to_server[function.name].server_label,
+            )
+            if error_exc:
+                message.error = str(error_exc)
+            elif (result and result.error_code and result.error_code > 0) or (result and result.error_message):
+                message.error = f"Error (code {result.error_code}): {result.error_message}"
+            elif result and result.content:
+                message.output = interleaved_content_as_str(result.content)
+        else:
+            if function.name == "web_search":
+                message = OpenAIResponseOutputMessageWebSearchToolCall(
+                    id=tool_call_id,
+                    status="completed",
+                )
+                if has_error:
+                    message.status = "failed"
+            elif function.name == "knowledge_search":
+                message = OpenAIResponseOutputMessageFileSearchToolCall(
+                    id=tool_call_id,
+                    queries=[tool_kwargs.get("query", "")],
+                    status="completed",
+                )
+                if result and "document_ids" in result.metadata:
+                    message.results = []
+                    for i, doc_id in enumerate(result.metadata["document_ids"]):
+                        text = result.metadata["chunks"][i] if "chunks" in result.metadata else None
+                        score = result.metadata["scores"][i] if "scores" in result.metadata else None
+                        message.results.append(
+                            OpenAIResponseOutputMessageFileSearchToolCallResults(
+                                file_id=doc_id,
+                                filename=doc_id,
+                                text=text,
+                                score=score,
+                                attributes={},
+                            )
+                        )
+                if has_error:
+                    message.status = "failed"
+            else:
+                raise ValueError(f"Unknown tool {function.name} called")
+
+        # Build input message
+        input_message = None
+        if result and result.content:
+            if isinstance(result.content, str):
+                content = result.content
+            elif isinstance(result.content, list):
+                content = []
+                for item in result.content:
+                    if isinstance(item, TextContentItem):
+                        part = OpenAIChatCompletionContentPartTextParam(text=item.text)
+                    elif isinstance(item, ImageContentItem):
+                        if item.image.data:
+                            url = f"data:image;base64,{item.image.data}"
+                        else:
+                            url = item.image.url
+                        part = OpenAIChatCompletionContentPartImageParam(image_url=OpenAIImageURL(url=url))
+                    else:
+                        raise ValueError(f"Unknown result content type: {type(item)}")
+                    content.append(part)
+            else:
+                raise ValueError(f"Unknown result content type: {type(result.content)}")
+            input_message = OpenAIToolMessageParam(content=content, tool_call_id=tool_call_id)
+        else:
+            text = str(error_exc) if error_exc else "Tool execution failed"
+            input_message = OpenAIToolMessageParam(content=text, tool_call_id=tool_call_id)
+
+        return message, input_message
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/types.py b/llama_stack/providers/inline/agents/meta_reference/responses/types.py
new file mode 100644
index 000000000..502d8a192
--- /dev/null
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/types.py
@@ -0,0 +1,62 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from dataclasses import dataclass
+
+from openai.types.chat import ChatCompletionToolParam
+from pydantic import BaseModel
+
+from llama_stack.apis.agents.openai_responses import (
+    OpenAIResponseInputTool,
+    OpenAIResponseInputToolMCP,
+    OpenAIResponseObjectStream,
+    OpenAIResponseOutput,
+)
+from llama_stack.apis.inference import OpenAIChatCompletionToolCall, OpenAIMessageParam, OpenAIResponseFormatParam
+
+
+class ToolExecutionResult(BaseModel):
+    """Result of streaming tool execution."""
+
+    stream_event: OpenAIResponseObjectStream | None = None
+    sequence_number: int
+    final_output_message: OpenAIResponseOutput | None = None
+    final_input_message: OpenAIMessageParam | None = None
+
+
+@dataclass
+class ChatCompletionResult:
+    """Result of processing streaming chat completion chunks."""
+
+    response_id: str
+    content: list[str]
+    tool_calls: dict[int, OpenAIChatCompletionToolCall]
+    created: int
+    model: str
+    finish_reason: str
+    message_item_id: str  # For streaming events
+    tool_call_item_ids: dict[int, str]  # For streaming events
+    content_part_emitted: bool  # Tracking state
+
+    @property
+    def content_text(self) -> str:
+        """Get joined content as string."""
+        return "".join(self.content)
+
+    @property
+    def has_tool_calls(self) -> bool:
+        """Check if there are any tool calls."""
+        return bool(self.tool_calls)
+
+
+class ChatCompletionContext(BaseModel):
+    model: str
+    messages: list[OpenAIMessageParam]
+    response_tools: list[OpenAIResponseInputTool] | None = None
+    chat_tools: list[ChatCompletionToolParam] | None = None
+    mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP]
+    temperature: float | None
+    response_format: OpenAIResponseFormatParam
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/utils.py b/llama_stack/providers/inline/agents/meta_reference/responses/utils.py
new file mode 100644
index 000000000..53d408ab7
--- /dev/null
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/utils.py
@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import uuid
+
+from llama_stack.apis.agents.openai_responses import (
+    OpenAIResponseInputTool,
+    OpenAIResponseMessage,
+    OpenAIResponseOutputMessageContentOutputText,
+)
+from llama_stack.apis.inference import (
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionToolCall,
+    OpenAIChoice,
+)
+
+
+async def convert_chat_choice_to_response_message(choice: OpenAIChoice) -> OpenAIResponseMessage:
+    """Convert an OpenAI Chat Completion choice into an OpenAI Response output message."""
+    output_content = ""
+    if isinstance(choice.message.content, str):
+        output_content = choice.message.content
+    elif isinstance(choice.message.content, OpenAIChatCompletionContentPartTextParam):
+        output_content = choice.message.content.text
+    else:
+        raise ValueError(
+            f"Llama Stack OpenAI Responses does not yet support output content type: {type(choice.message.content)}"
+        )
+
+    return OpenAIResponseMessage(
+        id=f"msg_{uuid.uuid4()}",
+        content=[OpenAIResponseOutputMessageContentOutputText(text=output_content)],
+        status="completed",
+        role="assistant",
+    )
+
+
+def is_function_tool_call(
+    tool_call: OpenAIChatCompletionToolCall,
+    tools: list[OpenAIResponseInputTool],
+) -> bool:
+    if not tool_call.function:
+        return False
+    for t in tools:
+        if t.type == "function" and t.name == tool_call.function.name:
+            return True
+    return False
diff --git a/tests/unit/providers/agents/meta_reference/test_openai_responses.py b/tests/unit/providers/agents/meta_reference/test_openai_responses.py
index 4132a74a3..5ea14d7c7 100644
--- a/tests/unit/providers/agents/meta_reference/test_openai_responses.py
+++ b/tests/unit/providers/agents/meta_reference/test_openai_responses.py
@@ -41,7 +41,7 @@ from llama_stack.apis.inference import (
 )
 from llama_stack.apis.tools.tools import Tool, ToolGroups, ToolInvocationResult, ToolParameter, ToolRuntime
 from llama_stack.core.access_control.access_control import default_policy
-from llama_stack.providers.inline.agents.meta_reference.openai_responses import (
+from llama_stack.providers.inline.agents.meta_reference.responses.openai_responses import (
     OpenAIResponsesImpl,
 )
 from llama_stack.providers.utils.responses.responses_store import ResponsesStore

From 9324e902f15f31f7087517afefef3bb2a59673b2 Mon Sep 17 00:00:00 2001
From: ashwinb <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 00:05:35 +0000
Subject: [PATCH 31/45] refactor(responses): move stuff into some utils and add
 unit tests (#3158)

# What does this PR do?
Refactors the OpenAI response conversion utilities by moving helper functions from `openai_responses.py` to `utils.py`. Adds unit tests.
---
 .../responses/openai_responses.py             | 126 +------
 .../agents/meta_reference/responses/utils.py  | 119 +++++++
 .../test_response_conversion_utils.py         | 310 ++++++++++++++++++
 3 files changed, 435 insertions(+), 120 deletions(-)
 create mode 100644 tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py

diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
index 9a87038bf..b586cf14c 100644
--- a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
@@ -19,9 +19,6 @@ from llama_stack.apis.agents.openai_responses import (
     MCPListToolsTool,
     OpenAIDeleteResponseObject,
     OpenAIResponseInput,
-    OpenAIResponseInputFunctionToolCallOutput,
-    OpenAIResponseInputMessageContent,
-    OpenAIResponseInputMessageContentImage,
     OpenAIResponseInputMessageContentText,
     OpenAIResponseInputTool,
     OpenAIResponseInputToolMCP,
@@ -29,9 +26,6 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseObject,
     OpenAIResponseObjectStream,
     OpenAIResponseOutput,
-    OpenAIResponseOutputMessageContent,
-    OpenAIResponseOutputMessageContentOutputText,
-    OpenAIResponseOutputMessageFunctionToolCall,
     OpenAIResponseOutputMessageMCPListTools,
     OpenAIResponseText,
     OpenAIResponseTextFormat,
@@ -39,23 +33,7 @@ from llama_stack.apis.agents.openai_responses import (
 )
 from llama_stack.apis.inference import (
     Inference,
-    OpenAIAssistantMessageParam,
-    OpenAIChatCompletionContentPartImageParam,
-    OpenAIChatCompletionContentPartParam,
-    OpenAIChatCompletionContentPartTextParam,
-    OpenAIChatCompletionToolCall,
-    OpenAIChatCompletionToolCallFunction,
-    OpenAIDeveloperMessageParam,
-    OpenAIImageURL,
-    OpenAIJSONSchema,
-    OpenAIMessageParam,
-    OpenAIResponseFormatJSONObject,
-    OpenAIResponseFormatJSONSchema,
-    OpenAIResponseFormatParam,
-    OpenAIResponseFormatText,
     OpenAISystemMessageParam,
-    OpenAIToolMessageParam,
-    OpenAIUserMessageParam,
 )
 from llama_stack.apis.tools import Tool, ToolGroups, ToolRuntime
 from llama_stack.apis.vector_io import VectorIO
@@ -69,106 +47,14 @@ from llama_stack.providers.utils.responses.responses_store import ResponsesStore
 from .streaming import StreamingResponseOrchestrator
 from .tool_executor import ToolExecutor
 from .types import ChatCompletionContext
+from .utils import (
+    convert_response_input_to_chat_messages,
+    convert_response_text_to_chat_response_format,
+)
 
 logger = get_logger(name=__name__, category="responses")
 
 
-async def _convert_response_content_to_chat_content(
-    content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
-) -> str | list[OpenAIChatCompletionContentPartParam]:
-    """
-    Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts.
-
-    The content schemas of each API look similar, but are not exactly the same.
-    """
-    if isinstance(content, str):
-        return content
-
-    converted_parts = []
-    for content_part in content:
-        if isinstance(content_part, OpenAIResponseInputMessageContentText):
-            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
-        elif isinstance(content_part, OpenAIResponseOutputMessageContentOutputText):
-            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
-        elif isinstance(content_part, OpenAIResponseInputMessageContentImage):
-            if content_part.image_url:
-                image_url = OpenAIImageURL(url=content_part.image_url, detail=content_part.detail)
-                converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url))
-        elif isinstance(content_part, str):
-            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part))
-        else:
-            raise ValueError(
-                f"Llama Stack OpenAI Responses does not yet support content type '{type(content_part)}' in this context"
-            )
-    return converted_parts
-
-
-async def _convert_response_input_to_chat_messages(
-    input: str | list[OpenAIResponseInput],
-) -> list[OpenAIMessageParam]:
-    """
-    Convert the input from an OpenAI Response API request into OpenAI Chat Completion messages.
-    """
-    messages: list[OpenAIMessageParam] = []
-    if isinstance(input, list):
-        for input_item in input:
-            if isinstance(input_item, OpenAIResponseInputFunctionToolCallOutput):
-                messages.append(
-                    OpenAIToolMessageParam(
-                        content=input_item.output,
-                        tool_call_id=input_item.call_id,
-                    )
-                )
-            elif isinstance(input_item, OpenAIResponseOutputMessageFunctionToolCall):
-                tool_call = OpenAIChatCompletionToolCall(
-                    index=0,
-                    id=input_item.call_id,
-                    function=OpenAIChatCompletionToolCallFunction(
-                        name=input_item.name,
-                        arguments=input_item.arguments,
-                    ),
-                )
-                messages.append(OpenAIAssistantMessageParam(tool_calls=[tool_call]))
-            else:
-                content = await _convert_response_content_to_chat_content(input_item.content)
-                message_type = await _get_message_type_by_role(input_item.role)
-                if message_type is None:
-                    raise ValueError(
-                        f"Llama Stack OpenAI Responses does not yet support message role '{input_item.role}' in this context"
-                    )
-                messages.append(message_type(content=content))
-    else:
-        messages.append(OpenAIUserMessageParam(content=input))
-    return messages
-
-
-async def _convert_response_text_to_chat_response_format(
-    text: OpenAIResponseText,
-) -> OpenAIResponseFormatParam:
-    """
-    Convert an OpenAI Response text parameter into an OpenAI Chat Completion response format.
-    """
-    if not text.format or text.format["type"] == "text":
-        return OpenAIResponseFormatText(type="text")
-    if text.format["type"] == "json_object":
-        return OpenAIResponseFormatJSONObject()
-    if text.format["type"] == "json_schema":
-        return OpenAIResponseFormatJSONSchema(
-            json_schema=OpenAIJSONSchema(name=text.format["name"], schema=text.format["schema"])
-        )
-    raise ValueError(f"Unsupported text format: {text.format}")
-
-
-async def _get_message_type_by_role(role: str):
-    role_to_type = {
-        "user": OpenAIUserMessageParam,
-        "system": OpenAISystemMessageParam,
-        "assistant": OpenAIAssistantMessageParam,
-        "developer": OpenAIDeveloperMessageParam,
-    }
-    return role_to_type.get(role)
-
-
 class OpenAIResponsePreviousResponseWithInputItems(BaseModel):
     input_items: ListOpenAIResponseInputItem
     response: OpenAIResponseObject
@@ -350,11 +236,11 @@ class OpenAIResponsesImpl:
     ) -> AsyncIterator[OpenAIResponseObjectStream]:
         # Input preprocessing
         input = await self._prepend_previous_response(input, previous_response_id)
-        messages = await _convert_response_input_to_chat_messages(input)
+        messages = await convert_response_input_to_chat_messages(input)
         await self._prepend_instructions(messages, instructions)
 
         # Structured outputs
-        response_format = await _convert_response_text_to_chat_response_format(text)
+        response_format = await convert_response_text_to_chat_response_format(text)
 
         # Tool setup, TODO: refactor this slightly since this can also yield events
         chat_tools, mcp_tool_to_server, mcp_list_message = (
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/utils.py b/llama_stack/providers/inline/agents/meta_reference/responses/utils.py
index 53d408ab7..1507a55c8 100644
--- a/llama_stack/providers/inline/agents/meta_reference/responses/utils.py
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/utils.py
@@ -7,14 +7,37 @@
 import uuid
 
 from llama_stack.apis.agents.openai_responses import (
+    OpenAIResponseInput,
+    OpenAIResponseInputFunctionToolCallOutput,
+    OpenAIResponseInputMessageContent,
+    OpenAIResponseInputMessageContentImage,
+    OpenAIResponseInputMessageContentText,
     OpenAIResponseInputTool,
     OpenAIResponseMessage,
+    OpenAIResponseOutputMessageContent,
     OpenAIResponseOutputMessageContentOutputText,
+    OpenAIResponseOutputMessageFunctionToolCall,
+    OpenAIResponseText,
 )
 from llama_stack.apis.inference import (
+    OpenAIAssistantMessageParam,
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartParam,
     OpenAIChatCompletionContentPartTextParam,
     OpenAIChatCompletionToolCall,
+    OpenAIChatCompletionToolCallFunction,
     OpenAIChoice,
+    OpenAIDeveloperMessageParam,
+    OpenAIImageURL,
+    OpenAIJSONSchema,
+    OpenAIMessageParam,
+    OpenAIResponseFormatJSONObject,
+    OpenAIResponseFormatJSONSchema,
+    OpenAIResponseFormatParam,
+    OpenAIResponseFormatText,
+    OpenAISystemMessageParam,
+    OpenAIToolMessageParam,
+    OpenAIUserMessageParam,
 )
 
 
@@ -38,6 +61,102 @@ async def convert_chat_choice_to_response_message(choice: OpenAIChoice) -> OpenA
     )
 
 
+async def convert_response_content_to_chat_content(
+    content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
+) -> str | list[OpenAIChatCompletionContentPartParam]:
+    """
+    Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts.
+
+    The content schemas of each API look similar, but are not exactly the same.
+    """
+    if isinstance(content, str):
+        return content
+
+    converted_parts = []
+    for content_part in content:
+        if isinstance(content_part, OpenAIResponseInputMessageContentText):
+            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
+        elif isinstance(content_part, OpenAIResponseOutputMessageContentOutputText):
+            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
+        elif isinstance(content_part, OpenAIResponseInputMessageContentImage):
+            if content_part.image_url:
+                image_url = OpenAIImageURL(url=content_part.image_url, detail=content_part.detail)
+                converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url))
+        elif isinstance(content_part, str):
+            converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part))
+        else:
+            raise ValueError(
+                f"Llama Stack OpenAI Responses does not yet support content type '{type(content_part)}' in this context"
+            )
+    return converted_parts
+
+
+async def convert_response_input_to_chat_messages(
+    input: str | list[OpenAIResponseInput],
+) -> list[OpenAIMessageParam]:
+    """
+    Convert the input from an OpenAI Response API request into OpenAI Chat Completion messages.
+    """
+    messages: list[OpenAIMessageParam] = []
+    if isinstance(input, list):
+        for input_item in input:
+            if isinstance(input_item, OpenAIResponseInputFunctionToolCallOutput):
+                messages.append(
+                    OpenAIToolMessageParam(
+                        content=input_item.output,
+                        tool_call_id=input_item.call_id,
+                    )
+                )
+            elif isinstance(input_item, OpenAIResponseOutputMessageFunctionToolCall):
+                tool_call = OpenAIChatCompletionToolCall(
+                    index=0,
+                    id=input_item.call_id,
+                    function=OpenAIChatCompletionToolCallFunction(
+                        name=input_item.name,
+                        arguments=input_item.arguments,
+                    ),
+                )
+                messages.append(OpenAIAssistantMessageParam(tool_calls=[tool_call]))
+            else:
+                content = await convert_response_content_to_chat_content(input_item.content)
+                message_type = await get_message_type_by_role(input_item.role)
+                if message_type is None:
+                    raise ValueError(
+                        f"Llama Stack OpenAI Responses does not yet support message role '{input_item.role}' in this context"
+                    )
+                messages.append(message_type(content=content))
+    else:
+        messages.append(OpenAIUserMessageParam(content=input))
+    return messages
+
+
+async def convert_response_text_to_chat_response_format(
+    text: OpenAIResponseText,
+) -> OpenAIResponseFormatParam:
+    """
+    Convert an OpenAI Response text parameter into an OpenAI Chat Completion response format.
+    """
+    if not text.format or text.format["type"] == "text":
+        return OpenAIResponseFormatText(type="text")
+    if text.format["type"] == "json_object":
+        return OpenAIResponseFormatJSONObject()
+    if text.format["type"] == "json_schema":
+        return OpenAIResponseFormatJSONSchema(
+            json_schema=OpenAIJSONSchema(name=text.format["name"], schema=text.format["schema"])
+        )
+    raise ValueError(f"Unsupported text format: {text.format}")
+
+
+async def get_message_type_by_role(role: str):
+    role_to_type = {
+        "user": OpenAIUserMessageParam,
+        "system": OpenAISystemMessageParam,
+        "assistant": OpenAIAssistantMessageParam,
+        "developer": OpenAIDeveloperMessageParam,
+    }
+    return role_to_type.get(role)
+
+
 def is_function_tool_call(
     tool_call: OpenAIChatCompletionToolCall,
     tools: list[OpenAIResponseInputTool],
diff --git a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py
new file mode 100644
index 000000000..b568ce135
--- /dev/null
+++ b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py
@@ -0,0 +1,310 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+
+import pytest
+
+from llama_stack.apis.agents.openai_responses import (
+    OpenAIResponseInputFunctionToolCallOutput,
+    OpenAIResponseInputMessageContentImage,
+    OpenAIResponseInputMessageContentText,
+    OpenAIResponseInputToolFunction,
+    OpenAIResponseInputToolWebSearch,
+    OpenAIResponseMessage,
+    OpenAIResponseOutputMessageContentOutputText,
+    OpenAIResponseOutputMessageFunctionToolCall,
+    OpenAIResponseText,
+    OpenAIResponseTextFormat,
+)
+from llama_stack.apis.inference import (
+    OpenAIAssistantMessageParam,
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionToolCall,
+    OpenAIChatCompletionToolCallFunction,
+    OpenAIChoice,
+    OpenAIDeveloperMessageParam,
+    OpenAIResponseFormatJSONObject,
+    OpenAIResponseFormatJSONSchema,
+    OpenAIResponseFormatText,
+    OpenAISystemMessageParam,
+    OpenAIToolMessageParam,
+    OpenAIUserMessageParam,
+)
+from llama_stack.providers.inline.agents.meta_reference.responses.utils import (
+    convert_chat_choice_to_response_message,
+    convert_response_content_to_chat_content,
+    convert_response_input_to_chat_messages,
+    convert_response_text_to_chat_response_format,
+    get_message_type_by_role,
+    is_function_tool_call,
+)
+
+
+class TestConvertChatChoiceToResponseMessage:
+    @pytest.mark.asyncio
+    async def test_convert_string_content(self):
+        choice = OpenAIChoice(
+            message=OpenAIAssistantMessageParam(content="Test message"),
+            finish_reason="stop",
+            index=0,
+        )
+
+        result = await convert_chat_choice_to_response_message(choice)
+
+        assert result.role == "assistant"
+        assert result.status == "completed"
+        assert len(result.content) == 1
+        assert isinstance(result.content[0], OpenAIResponseOutputMessageContentOutputText)
+        assert result.content[0].text == "Test message"
+
+    @pytest.mark.asyncio
+    async def test_convert_text_param_content(self):
+        choice = OpenAIChoice(
+            message=OpenAIAssistantMessageParam(
+                content=[OpenAIChatCompletionContentPartTextParam(text="Test text param")]
+            ),
+            finish_reason="stop",
+            index=0,
+        )
+
+        with pytest.raises(ValueError) as exc_info:
+            await convert_chat_choice_to_response_message(choice)
+
+        assert "does not yet support output content type" in str(exc_info.value)
+
+
+class TestConvertResponseContentToChatContent:
+    @pytest.mark.asyncio
+    async def test_convert_string_content(self):
+        result = await convert_response_content_to_chat_content("Simple string")
+        assert result == "Simple string"
+
+    @pytest.mark.asyncio
+    async def test_convert_text_content_parts(self):
+        content = [
+            OpenAIResponseInputMessageContentText(text="First part"),
+            OpenAIResponseOutputMessageContentOutputText(text="Second part"),
+        ]
+
+        result = await convert_response_content_to_chat_content(content)
+
+        assert len(result) == 2
+        assert isinstance(result[0], OpenAIChatCompletionContentPartTextParam)
+        assert result[0].text == "First part"
+        assert isinstance(result[1], OpenAIChatCompletionContentPartTextParam)
+        assert result[1].text == "Second part"
+
+    @pytest.mark.asyncio
+    async def test_convert_image_content(self):
+        content = [OpenAIResponseInputMessageContentImage(image_url="https://example.com/image.jpg", detail="high")]
+
+        result = await convert_response_content_to_chat_content(content)
+
+        assert len(result) == 1
+        assert isinstance(result[0], OpenAIChatCompletionContentPartImageParam)
+        assert result[0].image_url.url == "https://example.com/image.jpg"
+        assert result[0].image_url.detail == "high"
+
+
+class TestConvertResponseInputToChatMessages:
+    @pytest.mark.asyncio
+    async def test_convert_string_input(self):
+        result = await convert_response_input_to_chat_messages("User message")
+
+        assert len(result) == 1
+        assert isinstance(result[0], OpenAIUserMessageParam)
+        assert result[0].content == "User message"
+
+    @pytest.mark.asyncio
+    async def test_convert_function_tool_call_output(self):
+        input_items = [
+            OpenAIResponseInputFunctionToolCallOutput(
+                output="Tool output",
+                call_id="call_123",
+            )
+        ]
+
+        result = await convert_response_input_to_chat_messages(input_items)
+
+        assert len(result) == 1
+        assert isinstance(result[0], OpenAIToolMessageParam)
+        assert result[0].content == "Tool output"
+        assert result[0].tool_call_id == "call_123"
+
+    @pytest.mark.asyncio
+    async def test_convert_function_tool_call(self):
+        input_items = [
+            OpenAIResponseOutputMessageFunctionToolCall(
+                call_id="call_456",
+                name="test_function",
+                arguments='{"param": "value"}',
+            )
+        ]
+
+        result = await convert_response_input_to_chat_messages(input_items)
+
+        assert len(result) == 1
+        assert isinstance(result[0], OpenAIAssistantMessageParam)
+        assert len(result[0].tool_calls) == 1
+        assert result[0].tool_calls[0].id == "call_456"
+        assert result[0].tool_calls[0].function.name == "test_function"
+        assert result[0].tool_calls[0].function.arguments == '{"param": "value"}'
+
+    @pytest.mark.asyncio
+    async def test_convert_response_message(self):
+        input_items = [
+            OpenAIResponseMessage(
+                role="user",
+                content=[OpenAIResponseInputMessageContentText(text="User text")],
+            )
+        ]
+
+        result = await convert_response_input_to_chat_messages(input_items)
+
+        assert len(result) == 1
+        assert isinstance(result[0], OpenAIUserMessageParam)
+        # Content should be converted to chat content format
+        assert len(result[0].content) == 1
+        assert result[0].content[0].text == "User text"
+
+
+class TestConvertResponseTextToChatResponseFormat:
+    @pytest.mark.asyncio
+    async def test_convert_text_format(self):
+        text = OpenAIResponseText(format=OpenAIResponseTextFormat(type="text"))
+        result = await convert_response_text_to_chat_response_format(text)
+
+        assert isinstance(result, OpenAIResponseFormatText)
+        assert result.type == "text"
+
+    @pytest.mark.asyncio
+    async def test_convert_json_object_format(self):
+        text = OpenAIResponseText(format={"type": "json_object"})
+        result = await convert_response_text_to_chat_response_format(text)
+
+        assert isinstance(result, OpenAIResponseFormatJSONObject)
+
+    @pytest.mark.asyncio
+    async def test_convert_json_schema_format(self):
+        schema_def = {"type": "object", "properties": {"test": {"type": "string"}}}
+        text = OpenAIResponseText(
+            format={
+                "type": "json_schema",
+                "name": "test_schema",
+                "schema": schema_def,
+            }
+        )
+        result = await convert_response_text_to_chat_response_format(text)
+
+        assert isinstance(result, OpenAIResponseFormatJSONSchema)
+        assert result.json_schema["name"] == "test_schema"
+        assert result.json_schema["schema"] == schema_def
+
+    @pytest.mark.asyncio
+    async def test_default_text_format(self):
+        text = OpenAIResponseText()
+        result = await convert_response_text_to_chat_response_format(text)
+
+        assert isinstance(result, OpenAIResponseFormatText)
+        assert result.type == "text"
+
+
+class TestGetMessageTypeByRole:
+    @pytest.mark.asyncio
+    async def test_user_role(self):
+        result = await get_message_type_by_role("user")
+        assert result == OpenAIUserMessageParam
+
+    @pytest.mark.asyncio
+    async def test_system_role(self):
+        result = await get_message_type_by_role("system")
+        assert result == OpenAISystemMessageParam
+
+    @pytest.mark.asyncio
+    async def test_assistant_role(self):
+        result = await get_message_type_by_role("assistant")
+        assert result == OpenAIAssistantMessageParam
+
+    @pytest.mark.asyncio
+    async def test_developer_role(self):
+        result = await get_message_type_by_role("developer")
+        assert result == OpenAIDeveloperMessageParam
+
+    @pytest.mark.asyncio
+    async def test_unknown_role(self):
+        result = await get_message_type_by_role("unknown")
+        assert result is None
+
+
+class TestIsFunctionToolCall:
+    def test_is_function_tool_call_true(self):
+        tool_call = OpenAIChatCompletionToolCall(
+            index=0,
+            id="call_123",
+            function=OpenAIChatCompletionToolCallFunction(
+                name="test_function",
+                arguments="{}",
+            ),
+        )
+        tools = [
+            OpenAIResponseInputToolFunction(
+                type="function", name="test_function", parameters={"type": "object", "properties": {}}
+            ),
+            OpenAIResponseInputToolWebSearch(type="web_search"),
+        ]
+
+        result = is_function_tool_call(tool_call, tools)
+        assert result is True
+
+    def test_is_function_tool_call_false_different_name(self):
+        tool_call = OpenAIChatCompletionToolCall(
+            index=0,
+            id="call_123",
+            function=OpenAIChatCompletionToolCallFunction(
+                name="other_function",
+                arguments="{}",
+            ),
+        )
+        tools = [
+            OpenAIResponseInputToolFunction(
+                type="function", name="test_function", parameters={"type": "object", "properties": {}}
+            ),
+        ]
+
+        result = is_function_tool_call(tool_call, tools)
+        assert result is False
+
+    def test_is_function_tool_call_false_no_function(self):
+        tool_call = OpenAIChatCompletionToolCall(
+            index=0,
+            id="call_123",
+            function=None,
+        )
+        tools = [
+            OpenAIResponseInputToolFunction(
+                type="function", name="test_function", parameters={"type": "object", "properties": {}}
+            ),
+        ]
+
+        result = is_function_tool_call(tool_call, tools)
+        assert result is False
+
+    def test_is_function_tool_call_false_wrong_type(self):
+        tool_call = OpenAIChatCompletionToolCall(
+            index=0,
+            id="call_123",
+            function=OpenAIChatCompletionToolCallFunction(
+                name="web_search",
+                arguments="{}",
+            ),
+        )
+        tools = [
+            OpenAIResponseInputToolWebSearch(type="web_search"),
+        ]
+
+        result = is_function_tool_call(tool_call, tools)
+        assert result is False

From ba664474dec42ce45203319bcfb9d279ff2ad03d Mon Sep 17 00:00:00 2001
From: ashwinb <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 00:05:36 +0000
Subject: [PATCH 32/45] feat(responses): add mcp list tool streaming event
 (#3159)

# What does this PR do?

Adds proper streaming events for MCP tool listing (`mcp_list_tools.in_progress` and `mcp_list_tools.completed`). Also refactors things a bit more.

## Test Plan

Verified existing integration tests pass with the refactored code. The test `test_response_streaming_multi_turn_tool_execution` has been updated to check for the new MCP list tools streaming events
---
 .../responses/openai_responses.py             | 116 +---------
 .../meta_reference/responses/streaming.py     | 207 +++++++++++++++++-
 .../meta_reference/responses/tool_executor.py |  38 +++-
 .../agents/meta_reference/responses/types.py  |   2 -
 .../non_ci/responses/test_responses.py        |  42 +++-
 5 files changed, 260 insertions(+), 145 deletions(-)

diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
index b586cf14c..e528a4005 100644
--- a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py
@@ -8,40 +8,29 @@ import time
 import uuid
 from collections.abc import AsyncIterator
 
-from openai.types.chat import ChatCompletionToolParam
 from pydantic import BaseModel
 
 from llama_stack.apis.agents import Order
 from llama_stack.apis.agents.openai_responses import (
-    AllowedToolsFilter,
     ListOpenAIResponseInputItem,
     ListOpenAIResponseObject,
-    MCPListToolsTool,
     OpenAIDeleteResponseObject,
     OpenAIResponseInput,
     OpenAIResponseInputMessageContentText,
     OpenAIResponseInputTool,
-    OpenAIResponseInputToolMCP,
     OpenAIResponseMessage,
     OpenAIResponseObject,
     OpenAIResponseObjectStream,
-    OpenAIResponseOutput,
-    OpenAIResponseOutputMessageMCPListTools,
     OpenAIResponseText,
     OpenAIResponseTextFormat,
-    WebSearchToolTypes,
 )
 from llama_stack.apis.inference import (
     Inference,
     OpenAISystemMessageParam,
 )
-from llama_stack.apis.tools import Tool, ToolGroups, ToolRuntime
+from llama_stack.apis.tools import ToolGroups, ToolRuntime
 from llama_stack.apis.vector_io import VectorIO
 from llama_stack.log import get_logger
-from llama_stack.models.llama.datatypes import ToolDefinition, ToolParamDefinition
-from llama_stack.providers.utils.inference.openai_compat import (
-    convert_tooldef_to_openai_tool,
-)
 from llama_stack.providers.utils.responses.responses_store import ResponsesStore
 
 from .streaming import StreamingResponseOrchestrator
@@ -242,17 +231,10 @@ class OpenAIResponsesImpl:
         # Structured outputs
         response_format = await convert_response_text_to_chat_response_format(text)
 
-        # Tool setup, TODO: refactor this slightly since this can also yield events
-        chat_tools, mcp_tool_to_server, mcp_list_message = (
-            await self._convert_response_tools_to_chat_tools(tools) if tools else (None, {}, None)
-        )
-
         ctx = ChatCompletionContext(
             model=model,
             messages=messages,
             response_tools=tools,
-            chat_tools=chat_tools,
-            mcp_tool_to_server=mcp_tool_to_server,
             temperature=temperature,
             response_format=response_format,
         )
@@ -269,7 +251,6 @@ class OpenAIResponsesImpl:
             text=text,
             max_infer_iters=max_infer_iters,
             tool_executor=self.tool_executor,
-            mcp_list_message=mcp_list_message,
         )
 
         # Stream the response
@@ -288,98 +269,3 @@ class OpenAIResponsesImpl:
 
     async def delete_openai_response(self, response_id: str) -> OpenAIDeleteResponseObject:
         return await self.responses_store.delete_response_object(response_id)
-
-    async def _convert_response_tools_to_chat_tools(
-        self, tools: list[OpenAIResponseInputTool]
-    ) -> tuple[
-        list[ChatCompletionToolParam],
-        dict[str, OpenAIResponseInputToolMCP],
-        OpenAIResponseOutput | None,
-    ]:
-        mcp_tool_to_server = {}
-
-        def make_openai_tool(tool_name: str, tool: Tool) -> ChatCompletionToolParam:
-            tool_def = ToolDefinition(
-                tool_name=tool_name,
-                description=tool.description,
-                parameters={
-                    param.name: ToolParamDefinition(
-                        param_type=param.parameter_type,
-                        description=param.description,
-                        required=param.required,
-                        default=param.default,
-                    )
-                    for param in tool.parameters
-                },
-            )
-            return convert_tooldef_to_openai_tool(tool_def)
-
-        mcp_list_message = None
-        chat_tools: list[ChatCompletionToolParam] = []
-        for input_tool in tools:
-            # TODO: Handle other tool types
-            if input_tool.type == "function":
-                chat_tools.append(ChatCompletionToolParam(type="function", function=input_tool.model_dump()))
-            elif input_tool.type in WebSearchToolTypes:
-                tool_name = "web_search"
-                tool = await self.tool_groups_api.get_tool(tool_name)
-                if not tool:
-                    raise ValueError(f"Tool {tool_name} not found")
-                chat_tools.append(make_openai_tool(tool_name, tool))
-            elif input_tool.type == "file_search":
-                tool_name = "knowledge_search"
-                tool = await self.tool_groups_api.get_tool(tool_name)
-                if not tool:
-                    raise ValueError(f"Tool {tool_name} not found")
-                chat_tools.append(make_openai_tool(tool_name, tool))
-            elif input_tool.type == "mcp":
-                from llama_stack.providers.utils.tools.mcp import list_mcp_tools
-
-                always_allowed = None
-                never_allowed = None
-                if input_tool.allowed_tools:
-                    if isinstance(input_tool.allowed_tools, list):
-                        always_allowed = input_tool.allowed_tools
-                    elif isinstance(input_tool.allowed_tools, AllowedToolsFilter):
-                        always_allowed = input_tool.allowed_tools.always
-                        never_allowed = input_tool.allowed_tools.never
-
-                tool_defs = await list_mcp_tools(
-                    endpoint=input_tool.server_url,
-                    headers=input_tool.headers or {},
-                )
-
-                mcp_list_message = OpenAIResponseOutputMessageMCPListTools(
-                    id=f"mcp_list_{uuid.uuid4()}",
-                    status="completed",
-                    server_label=input_tool.server_label,
-                    tools=[],
-                )
-                for t in tool_defs.data:
-                    if never_allowed and t.name in never_allowed:
-                        continue
-                    if not always_allowed or t.name in always_allowed:
-                        chat_tools.append(make_openai_tool(t.name, t))
-                        if t.name in mcp_tool_to_server:
-                            raise ValueError(f"Duplicate tool name {t.name} found for server {input_tool.server_label}")
-                        mcp_tool_to_server[t.name] = input_tool
-                        mcp_list_message.tools.append(
-                            MCPListToolsTool(
-                                name=t.name,
-                                description=t.description,
-                                input_schema={
-                                    "type": "object",
-                                    "properties": {
-                                        p.name: {
-                                            "type": p.parameter_type,
-                                            "description": p.description,
-                                        }
-                                        for p in t.parameters
-                                    },
-                                    "required": [p.name for p in t.parameters if p.required],
-                                },
-                            )
-                        )
-            else:
-                raise ValueError(f"Llama Stack OpenAI Responses does not yet support tool type: {input_tool.type}")
-        return chat_tools, mcp_tool_to_server, mcp_list_message
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py b/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py
index 2e4ce0c37..0879e978a 100644
--- a/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py
@@ -9,7 +9,11 @@ from collections.abc import AsyncIterator
 from typing import Any
 
 from llama_stack.apis.agents.openai_responses import (
+    AllowedToolsFilter,
+    MCPListToolsTool,
     OpenAIResponseContentPartOutputText,
+    OpenAIResponseInputTool,
+    OpenAIResponseInputToolMCP,
     OpenAIResponseObject,
     OpenAIResponseObjectStream,
     OpenAIResponseObjectStreamResponseCompleted,
@@ -20,12 +24,16 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseObjectStreamResponseFunctionCallArgumentsDone,
     OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta,
     OpenAIResponseObjectStreamResponseMcpCallArgumentsDone,
+    OpenAIResponseObjectStreamResponseMcpListToolsCompleted,
+    OpenAIResponseObjectStreamResponseMcpListToolsInProgress,
     OpenAIResponseObjectStreamResponseOutputItemAdded,
     OpenAIResponseObjectStreamResponseOutputItemDone,
     OpenAIResponseObjectStreamResponseOutputTextDelta,
     OpenAIResponseOutput,
     OpenAIResponseOutputMessageFunctionToolCall,
+    OpenAIResponseOutputMessageMCPListTools,
     OpenAIResponseText,
+    WebSearchToolTypes,
 )
 from llama_stack.apis.inference import (
     Inference,
@@ -52,7 +60,6 @@ class StreamingResponseOrchestrator:
         text: OpenAIResponseText,
         max_infer_iters: int,
         tool_executor,  # Will be the tool execution logic from the main class
-        mcp_list_message: OpenAIResponseOutput | None = None,
     ):
         self.inference_api = inference_api
         self.ctx = ctx
@@ -62,13 +69,12 @@ class StreamingResponseOrchestrator:
         self.max_infer_iters = max_infer_iters
         self.tool_executor = tool_executor
         self.sequence_number = 0
-        self.mcp_list_message = mcp_list_message
+        # Store MCP tool mapping that gets built during tool processing
+        self.mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] = {}
 
     async def create_response(self) -> AsyncIterator[OpenAIResponseObjectStream]:
-        # Initialize output messages with MCP list message if present
+        # Initialize output messages
         output_messages: list[OpenAIResponseOutput] = []
-        if self.mcp_list_message:
-            output_messages.append(self.mcp_list_message)
         # Create initial response and emit response.created immediately
         initial_response = OpenAIResponseObject(
             created_at=self.created_at,
@@ -82,6 +88,11 @@ class StreamingResponseOrchestrator:
 
         yield OpenAIResponseObjectStreamResponseCreated(response=initial_response)
 
+        # Process all tools (including MCP tools) and emit streaming events
+        if self.ctx.response_tools:
+            async for stream_event in self._process_tools(self.ctx.response_tools, output_messages):
+                yield stream_event
+
         n_iter = 0
         messages = self.ctx.messages.copy()
 
@@ -261,9 +272,7 @@ class StreamingResponseOrchestrator:
                             self.sequence_number += 1
 
                             # Check if this is an MCP tool call
-                            is_mcp_tool = (
-                                tool_call.function.name and tool_call.function.name in self.ctx.mcp_tool_to_server
-                            )
+                            is_mcp_tool = tool_call.function.name and tool_call.function.name in self.mcp_tool_to_server
                             if is_mcp_tool:
                                 # Emit MCP-specific argument delta event
                                 yield OpenAIResponseObjectStreamResponseMcpCallArgumentsDelta(
@@ -294,9 +303,7 @@ class StreamingResponseOrchestrator:
             tool_call_name = chat_response_tool_calls[tool_call_index].function.name
 
             # Check if this is an MCP tool call
-            is_mcp_tool = (
-                self.ctx.mcp_tool_to_server and tool_call_name and tool_call_name in self.ctx.mcp_tool_to_server
-            )
+            is_mcp_tool = tool_call_name and tool_call_name in self.mcp_tool_to_server
             self.sequence_number += 1
             done_event_cls = (
                 OpenAIResponseObjectStreamResponseMcpCallArgumentsDone
@@ -391,7 +398,12 @@ class StreamingResponseOrchestrator:
             tool_call_log = None
             tool_response_message = None
             async for result in self.tool_executor.execute_tool_call(
-                tool_call, self.ctx, self.sequence_number, len(output_messages), matching_item_id
+                tool_call,
+                self.ctx,
+                self.sequence_number,
+                len(output_messages),
+                matching_item_id,
+                self.mcp_tool_to_server,
             ):
                 if result.stream_event:
                     # Forward streaming events
@@ -449,3 +461,174 @@ class StreamingResponseOrchestrator:
                 output_index=len(output_messages) - 1,
                 sequence_number=self.sequence_number,
             )
+
+    async def _process_tools(
+        self, tools: list[OpenAIResponseInputTool], output_messages: list[OpenAIResponseOutput]
+    ) -> AsyncIterator[OpenAIResponseObjectStream]:
+        """Process all tools and emit appropriate streaming events."""
+        from openai.types.chat import ChatCompletionToolParam
+
+        from llama_stack.apis.tools import Tool
+        from llama_stack.models.llama.datatypes import ToolDefinition, ToolParamDefinition
+        from llama_stack.providers.utils.inference.openai_compat import convert_tooldef_to_openai_tool
+
+        def make_openai_tool(tool_name: str, tool: Tool) -> ChatCompletionToolParam:
+            tool_def = ToolDefinition(
+                tool_name=tool_name,
+                description=tool.description,
+                parameters={
+                    param.name: ToolParamDefinition(
+                        param_type=param.parameter_type,
+                        description=param.description,
+                        required=param.required,
+                        default=param.default,
+                    )
+                    for param in tool.parameters
+                },
+            )
+            return convert_tooldef_to_openai_tool(tool_def)
+
+        # Initialize chat_tools if not already set
+        if self.ctx.chat_tools is None:
+            self.ctx.chat_tools = []
+
+        for input_tool in tools:
+            if input_tool.type == "function":
+                self.ctx.chat_tools.append(ChatCompletionToolParam(type="function", function=input_tool.model_dump()))
+            elif input_tool.type in WebSearchToolTypes:
+                tool_name = "web_search"
+                # Need to access tool_groups_api from tool_executor
+                tool = await self.tool_executor.tool_groups_api.get_tool(tool_name)
+                if not tool:
+                    raise ValueError(f"Tool {tool_name} not found")
+                self.ctx.chat_tools.append(make_openai_tool(tool_name, tool))
+            elif input_tool.type == "file_search":
+                tool_name = "knowledge_search"
+                tool = await self.tool_executor.tool_groups_api.get_tool(tool_name)
+                if not tool:
+                    raise ValueError(f"Tool {tool_name} not found")
+                self.ctx.chat_tools.append(make_openai_tool(tool_name, tool))
+            elif input_tool.type == "mcp":
+                async for stream_event in self._process_mcp_tool(input_tool, output_messages):
+                    yield stream_event
+            else:
+                raise ValueError(f"Llama Stack OpenAI Responses does not yet support tool type: {input_tool.type}")
+
+    async def _process_mcp_tool(
+        self, mcp_tool: OpenAIResponseInputToolMCP, output_messages: list[OpenAIResponseOutput]
+    ) -> AsyncIterator[OpenAIResponseObjectStream]:
+        """Process an MCP tool configuration and emit appropriate streaming events."""
+        from llama_stack.providers.utils.tools.mcp import list_mcp_tools
+
+        # Emit mcp_list_tools.in_progress
+        self.sequence_number += 1
+        yield OpenAIResponseObjectStreamResponseMcpListToolsInProgress(
+            sequence_number=self.sequence_number,
+        )
+
+        try:
+            # Parse allowed/never allowed tools
+            always_allowed = None
+            never_allowed = None
+            if mcp_tool.allowed_tools:
+                if isinstance(mcp_tool.allowed_tools, list):
+                    always_allowed = mcp_tool.allowed_tools
+                elif isinstance(mcp_tool.allowed_tools, AllowedToolsFilter):
+                    always_allowed = mcp_tool.allowed_tools.always
+                    never_allowed = mcp_tool.allowed_tools.never
+
+            # Call list_mcp_tools
+            tool_defs = await list_mcp_tools(
+                endpoint=mcp_tool.server_url,
+                headers=mcp_tool.headers or {},
+            )
+
+            # Create the MCP list tools message
+            mcp_list_message = OpenAIResponseOutputMessageMCPListTools(
+                id=f"mcp_list_{uuid.uuid4()}",
+                server_label=mcp_tool.server_label,
+                tools=[],
+            )
+
+            # Process tools and update context
+            for t in tool_defs.data:
+                if never_allowed and t.name in never_allowed:
+                    continue
+                if not always_allowed or t.name in always_allowed:
+                    # Add to chat tools for inference
+                    from llama_stack.models.llama.datatypes import ToolDefinition, ToolParamDefinition
+                    from llama_stack.providers.utils.inference.openai_compat import convert_tooldef_to_openai_tool
+
+                    tool_def = ToolDefinition(
+                        tool_name=t.name,
+                        description=t.description,
+                        parameters={
+                            param.name: ToolParamDefinition(
+                                param_type=param.parameter_type,
+                                description=param.description,
+                                required=param.required,
+                                default=param.default,
+                            )
+                            for param in t.parameters
+                        },
+                    )
+                    openai_tool = convert_tooldef_to_openai_tool(tool_def)
+                    if self.ctx.chat_tools is None:
+                        self.ctx.chat_tools = []
+                    self.ctx.chat_tools.append(openai_tool)
+
+                    # Add to MCP tool mapping
+                    if t.name in self.mcp_tool_to_server:
+                        raise ValueError(f"Duplicate tool name {t.name} found for server {mcp_tool.server_label}")
+                    self.mcp_tool_to_server[t.name] = mcp_tool
+
+                    # Add to MCP list message
+                    mcp_list_message.tools.append(
+                        MCPListToolsTool(
+                            name=t.name,
+                            description=t.description,
+                            input_schema={
+                                "type": "object",
+                                "properties": {
+                                    p.name: {
+                                        "type": p.parameter_type,
+                                        "description": p.description,
+                                    }
+                                    for p in t.parameters
+                                },
+                                "required": [p.name for p in t.parameters if p.required],
+                            },
+                        )
+                    )
+
+            # Add the MCP list message to output
+            output_messages.append(mcp_list_message)
+
+            # Emit output_item.added for the MCP list tools message
+            self.sequence_number += 1
+            yield OpenAIResponseObjectStreamResponseOutputItemAdded(
+                response_id=self.response_id,
+                item=mcp_list_message,
+                output_index=len(output_messages) - 1,
+                sequence_number=self.sequence_number,
+            )
+
+            # Emit mcp_list_tools.completed
+            self.sequence_number += 1
+            yield OpenAIResponseObjectStreamResponseMcpListToolsCompleted(
+                sequence_number=self.sequence_number,
+            )
+
+            # Emit output_item.done for the MCP list tools message
+            self.sequence_number += 1
+            yield OpenAIResponseObjectStreamResponseOutputItemDone(
+                response_id=self.response_id,
+                item=mcp_list_message,
+                output_index=len(output_messages) - 1,
+                sequence_number=self.sequence_number,
+            )
+
+        except Exception as e:
+            # TODO: Emit mcp_list_tools.failed event if needed
+            logger.exception(f"Failed to list MCP tools from {mcp_tool.server_url}: {e}")
+            raise
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py b/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py
index 6b7845138..5b98b4f51 100644
--- a/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py
@@ -10,6 +10,7 @@ from collections.abc import AsyncIterator
 
 from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseInputToolFileSearch,
+    OpenAIResponseInputToolMCP,
     OpenAIResponseObjectStreamResponseMcpCallCompleted,
     OpenAIResponseObjectStreamResponseMcpCallFailed,
     OpenAIResponseObjectStreamResponseMcpCallInProgress,
@@ -58,6 +59,7 @@ class ToolExecutor:
         sequence_number: int,
         output_index: int,
         item_id: str,
+        mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] | None = None,
     ) -> AsyncIterator[ToolExecutionResult]:
         tool_call_id = tool_call.id
         function = tool_call.function
@@ -69,25 +71,25 @@ class ToolExecutor:
 
         # Emit progress events for tool execution start
         async for event_result in self._emit_progress_events(
-            function.name, ctx, sequence_number, output_index, item_id
+            function.name, ctx, sequence_number, output_index, item_id, mcp_tool_to_server
         ):
             sequence_number = event_result.sequence_number
             yield event_result
 
         # Execute the actual tool call
-        error_exc, result = await self._execute_tool(function.name, tool_kwargs, ctx)
+        error_exc, result = await self._execute_tool(function.name, tool_kwargs, ctx, mcp_tool_to_server)
 
         # Emit completion events for tool execution
         has_error = error_exc or (result and ((result.error_code and result.error_code > 0) or result.error_message))
         async for event_result in self._emit_completion_events(
-            function.name, ctx, sequence_number, output_index, item_id, has_error
+            function.name, ctx, sequence_number, output_index, item_id, has_error, mcp_tool_to_server
         ):
             sequence_number = event_result.sequence_number
             yield event_result
 
         # Build result messages from tool execution
         output_message, input_message = await self._build_result_messages(
-            function, tool_call_id, tool_kwargs, ctx, error_exc, result, has_error
+            function, tool_call_id, tool_kwargs, ctx, error_exc, result, has_error, mcp_tool_to_server
         )
 
         # Yield the final result
@@ -161,12 +163,18 @@ class ToolExecutor:
         )
 
     async def _emit_progress_events(
-        self, function_name: str, ctx: ChatCompletionContext, sequence_number: int, output_index: int, item_id: str
+        self,
+        function_name: str,
+        ctx: ChatCompletionContext,
+        sequence_number: int,
+        output_index: int,
+        item_id: str,
+        mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] | None = None,
     ) -> AsyncIterator[ToolExecutionResult]:
         """Emit progress events for tool execution start."""
         # Emit in_progress event based on tool type (only for tools with specific streaming events)
         progress_event = None
-        if ctx.mcp_tool_to_server and function_name in ctx.mcp_tool_to_server:
+        if mcp_tool_to_server and function_name in mcp_tool_to_server:
             sequence_number += 1
             progress_event = OpenAIResponseObjectStreamResponseMcpCallInProgress(
                 item_id=item_id,
@@ -196,17 +204,21 @@ class ToolExecutor:
             yield ToolExecutionResult(stream_event=searching_event, sequence_number=sequence_number)
 
     async def _execute_tool(
-        self, function_name: str, tool_kwargs: dict, ctx: ChatCompletionContext
+        self,
+        function_name: str,
+        tool_kwargs: dict,
+        ctx: ChatCompletionContext,
+        mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] | None = None,
     ) -> tuple[Exception | None, any]:
         """Execute the tool and return error exception and result."""
         error_exc = None
         result = None
 
         try:
-            if ctx.mcp_tool_to_server and function_name in ctx.mcp_tool_to_server:
+            if mcp_tool_to_server and function_name in mcp_tool_to_server:
                 from llama_stack.providers.utils.tools.mcp import invoke_mcp_tool
 
-                mcp_tool = ctx.mcp_tool_to_server[function_name]
+                mcp_tool = mcp_tool_to_server[function_name]
                 result = await invoke_mcp_tool(
                     endpoint=mcp_tool.server_url,
                     headers=mcp_tool.headers or {},
@@ -244,11 +256,12 @@ class ToolExecutor:
         output_index: int,
         item_id: str,
         has_error: bool,
+        mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] | None = None,
     ) -> AsyncIterator[ToolExecutionResult]:
         """Emit completion or failure events for tool execution."""
         completion_event = None
 
-        if ctx.mcp_tool_to_server and function_name in ctx.mcp_tool_to_server:
+        if mcp_tool_to_server and function_name in mcp_tool_to_server:
             sequence_number += 1
             if has_error:
                 completion_event = OpenAIResponseObjectStreamResponseMcpCallFailed(
@@ -279,6 +292,7 @@ class ToolExecutor:
         error_exc: Exception | None,
         result: any,
         has_error: bool,
+        mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] | None = None,
     ) -> tuple[any, any]:
         """Build output and input messages from tool execution results."""
         from llama_stack.providers.utils.inference.prompt_adapter import (
@@ -286,7 +300,7 @@ class ToolExecutor:
         )
 
         # Build output message
-        if function.name in ctx.mcp_tool_to_server:
+        if mcp_tool_to_server and function.name in mcp_tool_to_server:
             from llama_stack.apis.agents.openai_responses import (
                 OpenAIResponseOutputMessageMCPCall,
             )
@@ -295,7 +309,7 @@ class ToolExecutor:
                 id=tool_call_id,
                 arguments=function.arguments,
                 name=function.name,
-                server_label=ctx.mcp_tool_to_server[function.name].server_label,
+                server_label=mcp_tool_to_server[function.name].server_label,
             )
             if error_exc:
                 message.error = str(error_exc)
diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/types.py b/llama_stack/providers/inline/agents/meta_reference/responses/types.py
index 502d8a192..89086c262 100644
--- a/llama_stack/providers/inline/agents/meta_reference/responses/types.py
+++ b/llama_stack/providers/inline/agents/meta_reference/responses/types.py
@@ -11,7 +11,6 @@ from pydantic import BaseModel
 
 from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseInputTool,
-    OpenAIResponseInputToolMCP,
     OpenAIResponseObjectStream,
     OpenAIResponseOutput,
 )
@@ -57,6 +56,5 @@ class ChatCompletionContext(BaseModel):
     messages: list[OpenAIMessageParam]
     response_tools: list[OpenAIResponseInputTool] | None = None
     chat_tools: list[ChatCompletionToolParam] | None = None
-    mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP]
     temperature: float | None
     response_format: OpenAIResponseFormatParam
diff --git a/tests/integration/non_ci/responses/test_responses.py b/tests/integration/non_ci/responses/test_responses.py
index 04266eec8..954f009c2 100644
--- a/tests/integration/non_ci/responses/test_responses.py
+++ b/tests/integration/non_ci/responses/test_responses.py
@@ -610,6 +610,14 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         mcp_in_progress_events = [chunk for chunk in chunks if chunk.type == "response.mcp_call.in_progress"]
         mcp_completed_events = [chunk for chunk in chunks if chunk.type == "response.mcp_call.completed"]
 
+        # Should have MCP list tools streaming events
+        mcp_list_tools_in_progress_events = [
+            chunk for chunk in chunks if chunk.type == "response.mcp_list_tools.in_progress"
+        ]
+        mcp_list_tools_completed_events = [
+            chunk for chunk in chunks if chunk.type == "response.mcp_list_tools.completed"
+        ]
+
         # Verify we have substantial streaming activity (not just batch events)
         assert len(chunks) > 10, f"Expected rich streaming with many events, got only {len(chunks)} chunks"
 
@@ -632,6 +640,14 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         assert len(mcp_completed_events) > 0, (
             f"Expected response.mcp_call.completed events, got chunk types: {chunk_types}"
         )
+
+        # Should have MCP list tools streaming events
+        assert len(mcp_list_tools_in_progress_events) > 0, (
+            f"Expected response.mcp_list_tools.in_progress events, got chunk types: {chunk_types}"
+        )
+        assert len(mcp_list_tools_completed_events) > 0, (
+            f"Expected response.mcp_list_tools.completed events, got chunk types: {chunk_types}"
+        )
         # MCP failed events are optional (only if errors occur)
 
         # Verify progress events have proper structure
@@ -643,6 +659,17 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
         for completed_event in mcp_completed_events:
             assert hasattr(completed_event, "sequence_number"), "Completed event should have 'sequence_number' field"
 
+        # Verify MCP list tools events have proper structure
+        for list_tools_progress_event in mcp_list_tools_in_progress_events:
+            assert hasattr(list_tools_progress_event, "sequence_number"), (
+                "MCP list tools progress event should have 'sequence_number' field"
+            )
+
+        for list_tools_completed_event in mcp_list_tools_completed_events:
+            assert hasattr(list_tools_completed_event, "sequence_number"), (
+                "MCP list tools completed event should have 'sequence_number' field"
+            )
+
         # Verify delta events have proper structure
         for delta_event in delta_events:
             assert hasattr(delta_event, "delta"), "Delta event should have 'delta' field"
@@ -662,8 +689,12 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
             assert hasattr(added_event, "output_index"), "Added event should have 'output_index' field"
             assert hasattr(added_event, "sequence_number"), "Added event should have 'sequence_number' field"
             assert hasattr(added_event, "response_id"), "Added event should have 'response_id' field"
-            assert added_event.item.type in ["function_call", "mcp_call"], "Added item should be a tool call"
-            assert added_event.item.status == "in_progress", "Added item should be in progress"
+            assert added_event.item.type in ["function_call", "mcp_call", "mcp_list_tools"], (
+                "Added item should be a tool call or MCP list tools"
+            )
+            if added_event.item.type in ["function_call", "mcp_call"]:
+                assert added_event.item.status == "in_progress", "Added tool call should be in progress"
+            # Note: mcp_list_tools doesn't have a status field, it's implicitly completed when added
             assert added_event.response_id, "Response ID should not be empty"
             assert isinstance(added_event.output_index, int), "Output index should be integer"
             assert added_event.output_index >= 0, "Output index should be non-negative"
@@ -674,10 +705,13 @@ def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_
             assert hasattr(done_event, "output_index"), "Done event should have 'output_index' field"
             assert hasattr(done_event, "sequence_number"), "Done event should have 'sequence_number' field"
             assert hasattr(done_event, "response_id"), "Done event should have 'response_id' field"
-            assert done_event.item.type in ["function_call", "mcp_call"], "Done item should be a tool call"
-            # Note: MCP calls don't have a status field, only function calls do
+            assert done_event.item.type in ["function_call", "mcp_call", "mcp_list_tools"], (
+                "Done item should be a tool call or MCP list tools"
+            )
+            # Note: MCP calls and mcp_list_tools don't have a status field, only function calls do
             if done_event.item.type == "function_call":
                 assert done_event.item.status == "completed", "Function call should be completed"
+            # Note: mcp_call and mcp_list_tools don't have status fields
             assert done_event.response_id, "Response ID should not be empty"
             assert isinstance(done_event.output_index, int), "Output index should be integer"
             assert done_event.output_index >= 0, "Output index should be non-negative"

From 8ed69978f9d08b146ba3d3d57cc5458fdd48bc54 Mon Sep 17 00:00:00 2001
From: ashwinb <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 00:05:36 +0000
Subject: [PATCH 33/45] refactor(tests): make the responses tests nicer (#3161)

# What does this PR do?

A _bunch_ on cleanup for the Responses tests.

- Got rid of YAML test cases, moved them to just use simple pydantic models
- Splitting the large monolithic test file into multiple focused test files:
   - `test_basic_responses.py` for basic and image response tests
   - `test_tool_responses.py` for tool-related tests
   - `test_file_search.py` for file search specific tests
- Adding a `StreamingValidator` helper class to standardize streaming response validation

## Test Plan

Run the tests:

```
pytest -s -v tests/integration/non_ci/responses/ \
   --stack-config=starter \
   --text-model openai/gpt-4o \
   --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
    -k "client_with_models"
```
---
 .../non_ci/responses/fixtures/fixtures.py     |   14 -
 .../non_ci/responses/fixtures/load.py         |   16 -
 .../non_ci/responses/fixtures/test_cases.py   |  262 ++++
 .../fixtures/test_cases/chat_completion.yaml  |  397 ------
 .../fixtures/test_cases/responses.yaml        |  166 ---
 tests/integration/non_ci/responses/helpers.py |   64 +
 .../non_ci/responses/streaming_assertions.py  |  145 +++
 .../non_ci/responses/test_basic_responses.py  |  188 +++
 .../non_ci/responses/test_file_search.py      |  318 +++++
 .../non_ci/responses/test_responses.py        | 1143 -----------------
 .../non_ci/responses/test_tool_responses.py   |  335 +++++
 11 files changed, 1312 insertions(+), 1736 deletions(-)
 delete mode 100644 tests/integration/non_ci/responses/fixtures/load.py
 create mode 100644 tests/integration/non_ci/responses/fixtures/test_cases.py
 delete mode 100644 tests/integration/non_ci/responses/fixtures/test_cases/chat_completion.yaml
 delete mode 100644 tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml
 create mode 100644 tests/integration/non_ci/responses/helpers.py
 create mode 100644 tests/integration/non_ci/responses/streaming_assertions.py
 create mode 100644 tests/integration/non_ci/responses/test_basic_responses.py
 create mode 100644 tests/integration/non_ci/responses/test_file_search.py
 delete mode 100644 tests/integration/non_ci/responses/test_responses.py
 create mode 100644 tests/integration/non_ci/responses/test_tool_responses.py

diff --git a/tests/integration/non_ci/responses/fixtures/fixtures.py b/tests/integration/non_ci/responses/fixtures/fixtures.py
index 2069010ad..62c4ae086 100644
--- a/tests/integration/non_ci/responses/fixtures/fixtures.py
+++ b/tests/integration/non_ci/responses/fixtures/fixtures.py
@@ -5,7 +5,6 @@
 # the root directory of this source tree.
 
 import os
-import re
 from pathlib import Path
 
 import pytest
@@ -48,19 +47,6 @@ def _load_all_verification_configs():
     return {"providers": all_provider_configs}
 
 
-def case_id_generator(case):
-    """Generate a test ID from the case's 'case_id' field, or use a default."""
-    case_id = case.get("case_id")
-    if isinstance(case_id, str | int):
-        return re.sub(r"\\W|^(?=\\d)", "_", str(case_id))
-    return None
-
-
-# Helper to get the base test name from the request object
-def get_base_test_name(request):
-    return request.node.originalname
-
-
 # --- End Helper Functions ---
 
 
diff --git a/tests/integration/non_ci/responses/fixtures/load.py b/tests/integration/non_ci/responses/fixtures/load.py
deleted file mode 100644
index 0184ee146..000000000
--- a/tests/integration/non_ci/responses/fixtures/load.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from pathlib import Path
-
-import yaml
-
-
-def load_test_cases(name: str):
-    fixture_dir = Path(__file__).parent / "test_cases"
-    yaml_path = fixture_dir / f"{name}.yaml"
-    with open(yaml_path) as f:
-        return yaml.safe_load(f)
diff --git a/tests/integration/non_ci/responses/fixtures/test_cases.py b/tests/integration/non_ci/responses/fixtures/test_cases.py
new file mode 100644
index 000000000..bdd1a5d81
--- /dev/null
+++ b/tests/integration/non_ci/responses/fixtures/test_cases.py
@@ -0,0 +1,262 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any
+
+import pytest
+from pydantic import BaseModel
+
+
+class ResponsesTestCase(BaseModel):
+    # Input can be a simple string or complex message structure
+    input: str | list[dict[str, Any]]
+    expected: str
+    # Tools as flexible dict structure (gets validated at runtime by the API)
+    tools: list[dict[str, Any]] | None = None
+    # Multi-turn conversations with input/output pairs
+    turns: list[tuple[str | list[dict[str, Any]], str]] | None = None
+    # File search specific fields
+    file_content: str | None = None
+    file_path: str | None = None
+    # Streaming flag
+    stream: bool | None = None
+
+
+# Basic response test cases
+basic_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="Which planet do humans live on?",
+            expected="earth",
+        ),
+        id="earth",
+    ),
+    pytest.param(
+        ResponsesTestCase(
+            input="Which planet has rings around it with a name starting with letter S?",
+            expected="saturn",
+        ),
+        id="saturn",
+    ),
+    pytest.param(
+        ResponsesTestCase(
+            input=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "input_text",
+                            "text": "what teams are playing in this image?",
+                        }
+                    ],
+                },
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "input_image",
+                            "image_url": "https://upload.wikimedia.org/wikipedia/commons/3/3b/LeBron_James_Layup_%28Cleveland_vs_Brooklyn_2018%29.jpg",
+                        }
+                    ],
+                },
+            ],
+            expected="brooklyn nets",
+        ),
+        id="image_input",
+    ),
+]
+
+# Multi-turn test cases
+multi_turn_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="",  # Not used for multi-turn
+            expected="",  # Not used for multi-turn
+            turns=[
+                ("Which planet do humans live on?", "earth"),
+                ("What is the name of the planet from your previous response?", "earth"),
+            ],
+        ),
+        id="earth",
+    ),
+]
+
+# Web search test cases
+web_search_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="How many experts does the Llama 4 Maverick model have?",
+            tools=[{"type": "web_search", "search_context_size": "low"}],
+            expected="128",
+        ),
+        id="llama_experts",
+    ),
+]
+
+# File search test cases
+file_search_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="How many experts does the Llama 4 Maverick model have?",
+            tools=[{"type": "file_search"}],
+            expected="128",
+            file_content="Llama 4 Maverick has 128 experts",
+        ),
+        id="llama_experts",
+    ),
+    pytest.param(
+        ResponsesTestCase(
+            input="How many experts does the Llama 4 Maverick model have?",
+            tools=[{"type": "file_search"}],
+            expected="128",
+            file_path="pdfs/llama_stack_and_models.pdf",
+        ),
+        id="llama_experts_pdf",
+    ),
+]
+
+# MCP tool test cases
+mcp_tool_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="What is the boiling point of myawesomeliquid in Celsius?",
+            tools=[{"type": "mcp", "server_label": "localmcp", "server_url": "<FILLED_BY_TEST_RUNNER>"}],
+            expected="Hello, world!",
+        ),
+        id="boiling_point_tool",
+    ),
+]
+
+# Custom tool test cases
+custom_tool_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="What's the weather like in San Francisco?",
+            tools=[
+                {
+                    "type": "function",
+                    "name": "get_weather",
+                    "description": "Get current temperature for a given location.",
+                    "parameters": {
+                        "additionalProperties": False,
+                        "properties": {
+                            "location": {
+                                "description": "City and country e.g. Bogotá, Colombia",
+                                "type": "string",
+                            }
+                        },
+                        "required": ["location"],
+                        "type": "object",
+                    },
+                }
+            ],
+            expected="",  # No specific expected output for custom tools
+        ),
+        id="sf_weather",
+    ),
+]
+
+# Image test cases
+image_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "input_text",
+                            "text": "Identify the type of animal in this image.",
+                        },
+                        {
+                            "type": "input_image",
+                            "image_url": "https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg",
+                        },
+                    ],
+                },
+            ],
+            expected="llama",
+        ),
+        id="llama_image",
+    ),
+]
+
+# Multi-turn image test cases
+multi_turn_image_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="",  # Not used for multi-turn
+            expected="",  # Not used for multi-turn
+            turns=[
+                (
+                    [
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "input_text",
+                                    "text": "What type of animal is in this image? Please respond with a single word that starts with the letter 'L'.",
+                                },
+                                {
+                                    "type": "input_image",
+                                    "image_url": "https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg",
+                                },
+                            ],
+                        },
+                    ],
+                    "llama",
+                ),
+                (
+                    "What country do you find this animal primarily in? What continent?",
+                    "peru",
+                ),
+            ],
+        ),
+        id="llama_image_understanding",
+    ),
+]
+
+# Multi-turn tool execution test cases
+multi_turn_tool_execution_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="I need to check if user 'alice' can access the file 'document.txt'. First, get alice's user ID, then check if that user ID can access the file 'document.txt'. Do this as a series of steps, where each step is a separate message. Return only one tool call per step. Summarize the final result with a single 'yes' or 'no' response.",
+            tools=[{"type": "mcp", "server_label": "localmcp", "server_url": "<FILLED_BY_TEST_RUNNER>"}],
+            expected="yes",
+        ),
+        id="user_file_access_check",
+    ),
+    pytest.param(
+        ResponsesTestCase(
+            input="I need to get the results for the 'boiling_point' experiment. First, get the experiment ID for 'boiling_point', then use that ID to get the experiment results. Tell me the boiling point in Celsius.",
+            tools=[{"type": "mcp", "server_label": "localmcp", "server_url": "<FILLED_BY_TEST_RUNNER>"}],
+            expected="100°C",
+        ),
+        id="experiment_results_lookup",
+    ),
+]
+
+# Multi-turn tool execution streaming test cases
+multi_turn_tool_execution_streaming_test_cases = [
+    pytest.param(
+        ResponsesTestCase(
+            input="Help me with this security check: First, get the user ID for 'charlie', then get the permissions for that user ID, and finally check if that user can access 'secret_file.txt'. Stream your progress as you work through each step. Return only one tool call per step. Summarize the final result with a single 'yes' or 'no' response.",
+            tools=[{"type": "mcp", "server_label": "localmcp", "server_url": "<FILLED_BY_TEST_RUNNER>"}],
+            expected="no",
+            stream=True,
+        ),
+        id="user_permissions_workflow",
+    ),
+    pytest.param(
+        ResponsesTestCase(
+            input="I need a complete analysis: First, get the experiment ID for 'chemical_reaction', then get the results for that experiment, and tell me if the yield was above 80%. Return only one tool call per step.  Please stream your analysis process.",
+            tools=[{"type": "mcp", "server_label": "localmcp", "server_url": "<FILLED_BY_TEST_RUNNER>"}],
+            expected="85%",
+            stream=True,
+        ),
+        id="experiment_analysis_streaming",
+    ),
+]
diff --git a/tests/integration/non_ci/responses/fixtures/test_cases/chat_completion.yaml b/tests/integration/non_ci/responses/fixtures/test_cases/chat_completion.yaml
deleted file mode 100644
index 0c9f1fe9e..000000000
--- a/tests/integration/non_ci/responses/fixtures/test_cases/chat_completion.yaml
+++ /dev/null
@@ -1,397 +0,0 @@
-test_chat_basic:
-  test_name: test_chat_basic
-  test_params:
-    case:
-    - case_id: "earth"
-      input:
-        messages:
-        - content: Which planet do humans live on?
-          role: user
-      output: Earth
-    - case_id: "saturn"
-      input:
-        messages:
-        - content: Which planet has rings around it with a name starting with letter
-            S?
-          role: user
-      output: Saturn
-test_chat_input_validation:
-  test_name: test_chat_input_validation
-  test_params:
-    case:
-    - case_id: "messages_missing"
-      input:
-        messages: []
-      output:
-        error:
-          status_code: 400
-    - case_id: "messages_role_invalid"
-      input:
-        messages:
-        - content: Which planet do humans live on?
-          role: fake_role
-      output:
-        error:
-          status_code: 400
-    - case_id: "tool_choice_invalid"
-      input:
-        messages:
-        - content: Which planet do humans live on?
-          role: user
-        tool_choice: invalid
-      output:
-        error:
-          status_code: 400
-    - case_id: "tool_choice_no_tools"
-      input:
-        messages:
-        - content: Which planet do humans live on?
-          role: user
-        tool_choice: required
-      output:
-        error:
-          status_code: 400
-    - case_id: "tools_type_invalid"
-      input:
-        messages:
-        - content: Which planet do humans live on?
-          role: user
-        tools:
-        - type: invalid
-      output:
-        error:
-          status_code: 400
-test_chat_image:
-  test_name: test_chat_image
-  test_params:
-    case:
-    - input:
-        messages:
-        - content:
-          - text: What is in this image?
-            type: text
-          - image_url:
-              url: https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg
-            type: image_url
-          role: user
-      output: llama
-test_chat_structured_output:
-  test_name: test_chat_structured_output
-  test_params:
-    case:
-    - case_id: "calendar"
-      input:
-        messages:
-        - content: Extract the event information.
-          role: system
-        - content: Alice and Bob are going to a science fair on Friday.
-          role: user
-        response_format:
-          json_schema:
-            name: calendar_event
-            schema:
-              properties:
-                date:
-                  title: Date
-                  type: string
-                name:
-                  title: Name
-                  type: string
-                participants:
-                  items:
-                    type: string
-                  title: Participants
-                  type: array
-              required:
-              - name
-              - date
-              - participants
-              title: CalendarEvent
-              type: object
-          type: json_schema
-      output: valid_calendar_event
-    - case_id: "math"
-      input:
-        messages:
-        - content: You are a helpful math tutor. Guide the user through the solution
-            step by step.
-          role: system
-        - content: how can I solve 8x + 7 = -23
-          role: user
-        response_format:
-          json_schema:
-            name: math_reasoning
-            schema:
-              $defs:
-                Step:
-                  properties:
-                    explanation:
-                      title: Explanation
-                      type: string
-                    output:
-                      title: Output
-                      type: string
-                  required:
-                  - explanation
-                  - output
-                  title: Step
-                  type: object
-              properties:
-                final_answer:
-                  title: Final Answer
-                  type: string
-                steps:
-                  items:
-                    $ref: '#/$defs/Step'
-                  title: Steps
-                  type: array
-              required:
-              - steps
-              - final_answer
-              title: MathReasoning
-              type: object
-          type: json_schema
-      output: valid_math_reasoning
-test_tool_calling:
-  test_name: test_tool_calling
-  test_params:
-    case:
-    - input:
-        messages:
-        - content: You are a helpful assistant that can use tools to get information.
-          role: system
-        - content: What's the weather like in San Francisco?
-          role: user
-        tools:
-        - function:
-            description: Get current temperature for a given location.
-            name: get_weather
-            parameters:
-              additionalProperties: false
-              properties:
-                location:
-                  description: "City and country e.g. Bogot\xE1, Colombia"
-                  type: string
-              required:
-              - location
-              type: object
-          type: function
-      output: get_weather_tool_call
-
-test_chat_multi_turn_tool_calling:
-  test_name: test_chat_multi_turn_tool_calling
-  test_params:
-    case:
-    - case_id: "text_then_weather_tool"
-      input:
-        messages:
-        - - role: user
-            content: "What's the name of the Sun in latin?"
-        - - role: user
-            content: "What's the weather like in San Francisco?"
-        tools:
-        - function:
-            description: Get the current weather
-            name: get_weather
-            parameters:
-              type: object
-              properties:
-                location:
-                  description: "The city and state (both required), e.g. San Francisco, CA."
-                  type: string
-              required: ["location"]
-          type: function
-      tool_responses:
-      - response: "{'response': '70 degrees and foggy'}"
-      expected:
-      - num_tool_calls: 0
-        answer: ["sol"]
-      - num_tool_calls: 1
-        tool_name: get_weather
-        tool_arguments:
-          location: "San Francisco, CA"
-      - num_tool_calls: 0
-        answer: ["foggy", "70 degrees"]
-    - case_id: "weather_tool_then_text"
-      input:
-        messages:
-        - - role: user
-            content: "What's the weather like in San Francisco?"
-        tools:
-        - function:
-            description: Get the current weather
-            name: get_weather
-            parameters:
-              type: object
-              properties:
-                location:
-                  description: "The city and state (both required), e.g. San Francisco, CA."
-                  type: string
-              required: ["location"]
-          type: function
-      tool_responses:
-      - response: "{'response': '70 degrees and foggy'}"
-      expected:
-      - num_tool_calls: 1
-        tool_name: get_weather
-        tool_arguments:
-          location: "San Francisco, CA"
-      - num_tool_calls: 0
-        answer: ["foggy", "70 degrees"]
-    - case_id: "add_product_tool"
-      input:
-        messages:
-        - - role: user
-            content: "Please add a new product with name 'Widget', price 19.99, in stock, and tags ['new', 'sale'] and give me the product id."
-        tools:
-        - function:
-            description: Add a new product
-            name: addProduct
-            parameters:
-              type: object
-              properties:
-                name:
-                  description: "Name of the product"
-                  type: string
-                price:
-                  description: "Price of the product"
-                  type: number
-                inStock:
-                  description: "Availability status of the product."
-                  type: boolean
-                tags:
-                  description: "List of product tags"
-                  type: array
-                  items:
-                    type: string
-              required: ["name", "price", "inStock"]
-          type: function
-      tool_responses:
-      - response: "{'response': 'Successfully added product with id: 123'}"
-      expected:
-      - num_tool_calls: 1
-        tool_name: addProduct
-        tool_arguments:
-          name: "Widget"
-          price: 19.99
-          inStock: true
-          tags:
-          - "new"
-          - "sale"
-      - num_tool_calls: 0
-        answer: ["123", "product id: 123"]
-    - case_id: "get_then_create_event_tool"
-      input:
-        messages:
-        - - role: system
-            content: "Todays date is 2025-03-01."
-          - role: user
-            content: "Do i have any meetings on March 3rd at 10 am? Yes or no?"
-        - - role: user
-            content: "Alright then, Create an event named 'Team Building', scheduled for that time same time, in the 'Main Conference Room' and add Alice, Bob, Charlie to it. Give me the created event id."
-        tools:
-        - function:
-            description: Create a new event
-            name: create_event
-            parameters:
-              type: object
-              properties:
-                name:
-                  description: "Name of the event"
-                  type: string
-                date:
-                  description: "Date of the event in ISO format"
-                  type: string
-                time:
-                  description: "Event Time (HH:MM)"
-                  type: string
-                location:
-                  description: "Location of the event"
-                  type: string
-                participants:
-                  description: "List of participant names"
-                  type: array
-                  items:
-                    type: string
-              required: ["name", "date", "time", "location", "participants"]
-          type: function
-        - function:
-            description: Get an event by date and time
-            name: get_event
-            parameters:
-              type: object
-              properties:
-                date:
-                  description: "Date of the event in ISO format"
-                  type: string
-                time:
-                  description: "Event Time (HH:MM)"
-                  type: string
-              required: ["date", "time"]
-          type: function
-      tool_responses:
-      - response: "{'response': 'No events found for 2025-03-03 at 10:00'}"
-      - response: "{'response': 'Successfully created new event with id: e_123'}"
-      expected:
-      - num_tool_calls: 1
-        tool_name: get_event
-        tool_arguments:
-          date: "2025-03-03"
-          time: "10:00"
-      - num_tool_calls: 0
-        answer: ["no", "no events found", "no meetings"]
-      - num_tool_calls: 1
-        tool_name: create_event
-        tool_arguments:
-          name: "Team Building"
-          date: "2025-03-03"
-          time: "10:00"
-          location: "Main Conference Room"
-          participants:
-          - "Alice"
-          - "Bob"
-          - "Charlie"
-      - num_tool_calls: 0
-        answer: ["e_123", "event id: e_123"]
-    - case_id: "compare_monthly_expense_tool"
-      input:
-        messages:
-        - - role: system
-            content: "Todays date is 2025-03-01."
-          - role: user
-            content: "what was my monthly expense in Jan of this year?"
-        - - role: user
-            content: "Was it less than Feb of last year? Only answer with yes or no."
-        tools:
-        - function:
-            description: Get monthly expense summary
-            name: getMonthlyExpenseSummary
-            parameters:
-              type: object
-              properties:
-                month:
-                  description: "Month of the year (1-12)"
-                  type: integer
-                year:
-                  description: "Year"
-                  type: integer
-              required: ["month", "year"]
-          type: function
-      tool_responses:
-      - response: "{'response': 'Total expenses for January 2025: $1000'}"
-      - response: "{'response': 'Total expenses for February 2024: $2000'}"
-      expected:
-      - num_tool_calls: 1
-        tool_name: getMonthlyExpenseSummary
-        tool_arguments:
-          month: 1
-          year: 2025
-      - num_tool_calls: 0
-        answer: ["1000", "$1,000", "1,000"]
-      - num_tool_calls: 1
-        tool_name: getMonthlyExpenseSummary
-        tool_arguments:
-          month: 2
-          year: 2024
-      - num_tool_calls: 0
-        answer: ["yes"]
diff --git a/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml b/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml
deleted file mode 100644
index 353a64291..000000000
--- a/tests/integration/non_ci/responses/fixtures/test_cases/responses.yaml
+++ /dev/null
@@ -1,166 +0,0 @@
-test_response_basic:
-  test_name: test_response_basic
-  test_params:
-    case:
-    - case_id: "earth"
-      input: "Which planet do humans live on?"
-      output: "earth"
-    - case_id: "saturn"
-      input: "Which planet has rings around it with a name starting with letter S?"
-      output: "saturn"
-    - case_id: "image_input"
-      input:
-      - role: user
-        content:
-        - type: input_text
-          text: "what teams are playing in this image?"
-      - role: user
-        content:
-        - type: input_image
-          image_url: "https://upload.wikimedia.org/wikipedia/commons/3/3b/LeBron_James_Layup_%28Cleveland_vs_Brooklyn_2018%29.jpg"
-      output: "brooklyn nets"
-
-test_response_multi_turn:
-  test_name: test_response_multi_turn
-  test_params:
-    case:
-    - case_id: "earth"
-      turns:
-      - input: "Which planet do humans live on?"
-        output: "earth"
-      - input: "What is the name of the planet from your previous response?"
-        output: "earth"
-
-test_response_web_search:
-  test_name: test_response_web_search
-  test_params:
-    case:
-    - case_id: "llama_experts"
-      input: "How many experts does the Llama 4 Maverick model have?"
-      tools:
-      - type: web_search
-        search_context_size: "low"
-      output: "128"
-
-test_response_file_search:
-  test_name: test_response_file_search
-  test_params:
-    case:
-    - case_id: "llama_experts"
-      input: "How many experts does the Llama 4 Maverick model have?"
-      tools:
-      - type: file_search
-        # vector_store_ids param for file_search tool gets added by the test runner
-      file_content: "Llama 4 Maverick has 128 experts"
-      output: "128"
-    - case_id: "llama_experts_pdf"
-      input: "How many experts does the Llama 4 Maverick model have?"
-      tools:
-      - type: file_search
-        # vector_store_ids param for file_search toolgets added by the test runner
-      file_path: "pdfs/llama_stack_and_models.pdf"
-      output: "128"
-
-test_response_mcp_tool:
-  test_name: test_response_mcp_tool
-  test_params:
-    case:
-    - case_id: "boiling_point_tool"
-      input: "What is the boiling point of myawesomeliquid in Celsius?"
-      tools:
-      - type: mcp
-        server_label: "localmcp"
-        server_url: "<FILLED_BY_TEST_RUNNER>"
-      output: "Hello, world!"
-
-test_response_custom_tool:
-  test_name: test_response_custom_tool
-  test_params:
-    case:
-    - case_id: "sf_weather"
-      input: "What's the weather like in San Francisco?"
-      tools:
-      - type: function
-        name: get_weather
-        description: Get current temperature for a given location.
-        parameters:
-          additionalProperties: false
-          properties:
-            location:
-              description: "City and country e.g. Bogot\xE1, Colombia"
-              type: string
-          required:
-          - location
-          type: object
-
-test_response_image:
-  test_name: test_response_image
-  test_params:
-    case:
-    - case_id: "llama_image"
-      input:
-      - role: user
-        content:
-        - type: input_text
-          text: "Identify the type of animal in this image."
-        - type: input_image
-          image_url: "https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg"
-      output: "llama"
-
-# the models are really poor at tool calling after seeing images :/
-test_response_multi_turn_image:
-  test_name: test_response_multi_turn_image
-  test_params:
-    case:
-    - case_id: "llama_image_understanding"
-      turns:
-      - input:
-        - role: user
-          content:
-          - type: input_text
-            text: "What type of animal is in this image? Please respond with a single word that starts with the letter 'L'."
-          - type: input_image
-            image_url: "https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg"
-        output: "llama"
-      - input: "What country do you find this animal primarily in? What continent?"
-        output: "peru"
-
-test_response_multi_turn_tool_execution:
-  test_name: test_response_multi_turn_tool_execution
-  test_params:
-    case:
-    - case_id: "user_file_access_check"
-      input: "I need to check if user 'alice' can access the file 'document.txt'. First, get alice's user ID, then check if that user ID can access the file 'document.txt'. Do this as a series of steps, where each step is a separate message. Return only one tool call per step. Summarize the final result with a single 'yes' or 'no' response."
-      tools:
-      - type: mcp
-        server_label: "localmcp"
-        server_url: "<FILLED_BY_TEST_RUNNER>"
-      output: "yes"
-    - case_id: "experiment_results_lookup"
-      input: "I need to get the results for the 'boiling_point' experiment. First, get the experiment ID for 'boiling_point', then use that ID to get the experiment results. Tell me the boiling point in Celsius."
-      tools:
-      - type: mcp
-        server_label: "localmcp"
-        server_url: "<FILLED_BY_TEST_RUNNER>"
-      output: "100°C"
-
-test_response_multi_turn_tool_execution_streaming:
-  test_name: test_response_multi_turn_tool_execution_streaming
-  test_params:
-    case:
-    - case_id: "user_permissions_workflow"
-      input: "Help me with this security check: First, get the user ID for 'charlie', then get the permissions for that user ID, and finally check if that user can access 'secret_file.txt'. Stream your progress as you work through each step. Return only one tool call per step. Summarize the final result with a single 'yes' or 'no' response."
-      tools:
-      - type: mcp
-        server_label: "localmcp"
-        server_url: "<FILLED_BY_TEST_RUNNER>"
-      stream: true
-      output: "no"
-    - case_id: "experiment_analysis_streaming"
-      input: "I need a complete analysis: First, get the experiment ID for 'chemical_reaction', then get the results for that experiment, and tell me if the yield was above 80%. Return only one tool call per step.  Please stream your analysis process."
-      tools:
-      - type: mcp
-        server_label: "localmcp"
-        server_url: "<FILLED_BY_TEST_RUNNER>"
-      stream: true
-      output: "85%"
diff --git a/tests/integration/non_ci/responses/helpers.py b/tests/integration/non_ci/responses/helpers.py
new file mode 100644
index 000000000..7c988402f
--- /dev/null
+++ b/tests/integration/non_ci/responses/helpers.py
@@ -0,0 +1,64 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import time
+
+
+def new_vector_store(openai_client, name):
+    """Create a new vector store, cleaning up any existing one with the same name."""
+    # Ensure we don't reuse an existing vector store
+    vector_stores = openai_client.vector_stores.list()
+    for vector_store in vector_stores:
+        if vector_store.name == name:
+            openai_client.vector_stores.delete(vector_store_id=vector_store.id)
+
+    # Create a new vector store
+    vector_store = openai_client.vector_stores.create(name=name)
+    return vector_store
+
+
+def upload_file(openai_client, name, file_path):
+    """Upload a file, cleaning up any existing file with the same name."""
+    # Ensure we don't reuse an existing file
+    files = openai_client.files.list()
+    for file in files:
+        if file.filename == name:
+            openai_client.files.delete(file_id=file.id)
+
+    # Upload a text file with our document content
+    return openai_client.files.create(file=open(file_path, "rb"), purpose="assistants")
+
+
+def wait_for_file_attachment(compat_client, vector_store_id, file_id):
+    """Wait for a file to be attached to a vector store."""
+    file_attach_response = compat_client.vector_stores.files.retrieve(
+        vector_store_id=vector_store_id,
+        file_id=file_id,
+    )
+
+    while file_attach_response.status == "in_progress":
+        time.sleep(0.1)
+        file_attach_response = compat_client.vector_stores.files.retrieve(
+            vector_store_id=vector_store_id,
+            file_id=file_id,
+        )
+
+    assert file_attach_response.status == "completed", f"Expected file to be attached, got {file_attach_response}"
+    assert not file_attach_response.last_error
+    return file_attach_response
+
+
+def setup_mcp_tools(tools, mcp_server_info):
+    """Replace placeholder MCP server URLs with actual server info."""
+    # Create a deep copy to avoid modifying the original test case
+    import copy
+
+    tools_copy = copy.deepcopy(tools)
+
+    for tool in tools_copy:
+        if tool["type"] == "mcp" and tool["server_url"] == "<FILLED_BY_TEST_RUNNER>":
+            tool["server_url"] = mcp_server_info["server_url"]
+    return tools_copy
diff --git a/tests/integration/non_ci/responses/streaming_assertions.py b/tests/integration/non_ci/responses/streaming_assertions.py
new file mode 100644
index 000000000..4279ffbab
--- /dev/null
+++ b/tests/integration/non_ci/responses/streaming_assertions.py
@@ -0,0 +1,145 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any
+
+
+class StreamingValidator:
+    """Helper class for validating streaming response events."""
+
+    def __init__(self, chunks: list[Any]):
+        self.chunks = chunks
+        self.event_types = [chunk.type for chunk in chunks]
+
+    def assert_basic_event_sequence(self):
+        """Verify basic created -> completed event sequence."""
+        assert len(self.chunks) >= 2, f"Expected at least 2 chunks (created + completed), got {len(self.chunks)}"
+        assert self.chunks[0].type == "response.created", (
+            f"First chunk should be response.created, got {self.chunks[0].type}"
+        )
+        assert self.chunks[-1].type == "response.completed", (
+            f"Last chunk should be response.completed, got {self.chunks[-1].type}"
+        )
+
+        # Verify event order
+        created_index = self.event_types.index("response.created")
+        completed_index = self.event_types.index("response.completed")
+        assert created_index < completed_index, "response.created should come before response.completed"
+
+    def assert_response_consistency(self):
+        """Verify response ID consistency across events."""
+        response_ids = set()
+        for chunk in self.chunks:
+            if hasattr(chunk, "response_id"):
+                response_ids.add(chunk.response_id)
+            elif hasattr(chunk, "response") and hasattr(chunk.response, "id"):
+                response_ids.add(chunk.response.id)
+
+        assert len(response_ids) == 1, f"All events should reference the same response_id, found: {response_ids}"
+
+    def assert_has_incremental_content(self):
+        """Verify that content is delivered incrementally via delta events."""
+        delta_events = [
+            i for i, event_type in enumerate(self.event_types) if event_type == "response.output_text.delta"
+        ]
+        assert len(delta_events) > 0, "Expected delta events for true incremental streaming, but found none"
+
+        # Verify delta events have content
+        non_empty_deltas = 0
+        delta_content_total = ""
+
+        for delta_idx in delta_events:
+            chunk = self.chunks[delta_idx]
+            if hasattr(chunk, "delta") and chunk.delta:
+                delta_content_total += chunk.delta
+                non_empty_deltas += 1
+
+        assert non_empty_deltas > 0, "Delta events found but none contain content"
+        assert len(delta_content_total) > 0, "Delta events found but total delta content is empty"
+
+        return delta_content_total
+
+    def assert_content_quality(self, expected_content: str):
+        """Verify the final response contains expected content."""
+        final_chunk = self.chunks[-1]
+        if hasattr(final_chunk, "response"):
+            output_text = final_chunk.response.output_text.lower().strip()
+            assert len(output_text) > 0, "Response should have content"
+            assert expected_content.lower() in output_text, f"Expected '{expected_content}' in response"
+
+    def assert_has_tool_calls(self):
+        """Verify tool call streaming events are present."""
+        # Check for tool call events
+        delta_events = [
+            chunk
+            for chunk in self.chunks
+            if chunk.type in ["response.function_call_arguments.delta", "response.mcp_call.arguments.delta"]
+        ]
+        done_events = [
+            chunk
+            for chunk in self.chunks
+            if chunk.type in ["response.function_call_arguments.done", "response.mcp_call.arguments.done"]
+        ]
+
+        assert len(delta_events) > 0, f"Expected tool call delta events, got chunk types: {self.event_types}"
+        assert len(done_events) > 0, f"Expected tool call done events, got chunk types: {self.event_types}"
+
+        # Verify output item events
+        item_added_events = [chunk for chunk in self.chunks if chunk.type == "response.output_item.added"]
+        item_done_events = [chunk for chunk in self.chunks if chunk.type == "response.output_item.done"]
+
+        assert len(item_added_events) > 0, (
+            f"Expected response.output_item.added events, got chunk types: {self.event_types}"
+        )
+        assert len(item_done_events) > 0, (
+            f"Expected response.output_item.done events, got chunk types: {self.event_types}"
+        )
+
+    def assert_has_mcp_events(self):
+        """Verify MCP-specific streaming events are present."""
+        # Tool execution progress events
+        mcp_in_progress_events = [chunk for chunk in self.chunks if chunk.type == "response.mcp_call.in_progress"]
+        mcp_completed_events = [chunk for chunk in self.chunks if chunk.type == "response.mcp_call.completed"]
+
+        assert len(mcp_in_progress_events) > 0, (
+            f"Expected response.mcp_call.in_progress events, got chunk types: {self.event_types}"
+        )
+        assert len(mcp_completed_events) > 0, (
+            f"Expected response.mcp_call.completed events, got chunk types: {self.event_types}"
+        )
+
+        # MCP list tools events
+        mcp_list_tools_in_progress_events = [
+            chunk for chunk in self.chunks if chunk.type == "response.mcp_list_tools.in_progress"
+        ]
+        mcp_list_tools_completed_events = [
+            chunk for chunk in self.chunks if chunk.type == "response.mcp_list_tools.completed"
+        ]
+
+        assert len(mcp_list_tools_in_progress_events) > 0, (
+            f"Expected response.mcp_list_tools.in_progress events, got chunk types: {self.event_types}"
+        )
+        assert len(mcp_list_tools_completed_events) > 0, (
+            f"Expected response.mcp_list_tools.completed events, got chunk types: {self.event_types}"
+        )
+
+    def assert_rich_streaming(self, min_chunks: int = 10):
+        """Verify we have substantial streaming activity."""
+        assert len(self.chunks) > min_chunks, (
+            f"Expected rich streaming with many events, got only {len(self.chunks)} chunks"
+        )
+
+    def validate_event_structure(self):
+        """Validate the structure of various event types."""
+        for chunk in self.chunks:
+            if chunk.type == "response.created":
+                assert chunk.response.status == "in_progress"
+            elif chunk.type == "response.completed":
+                assert chunk.response.status == "completed"
+            elif hasattr(chunk, "item_id"):
+                assert chunk.item_id, "Events with item_id should have non-empty item_id"
+            elif hasattr(chunk, "sequence_number"):
+                assert isinstance(chunk.sequence_number, int), "sequence_number should be an integer"
diff --git a/tests/integration/non_ci/responses/test_basic_responses.py b/tests/integration/non_ci/responses/test_basic_responses.py
new file mode 100644
index 000000000..a8106e593
--- /dev/null
+++ b/tests/integration/non_ci/responses/test_basic_responses.py
@@ -0,0 +1,188 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import time
+
+import pytest
+from fixtures.test_cases import basic_test_cases, image_test_cases, multi_turn_image_test_cases, multi_turn_test_cases
+from streaming_assertions import StreamingValidator
+
+
+@pytest.mark.parametrize("case", basic_test_cases)
+def test_response_non_streaming_basic(compat_client, text_model_id, case):
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        stream=False,
+    )
+    output_text = response.output_text.lower().strip()
+    assert len(output_text) > 0
+    assert case.expected.lower() in output_text
+
+    retrieved_response = compat_client.responses.retrieve(response_id=response.id)
+    assert retrieved_response.output_text == response.output_text
+
+    next_response = compat_client.responses.create(
+        model=text_model_id,
+        input="Repeat your previous response in all caps.",
+        previous_response_id=response.id,
+    )
+    next_output_text = next_response.output_text.strip()
+    assert case.expected.upper() in next_output_text
+
+
+@pytest.mark.parametrize("case", basic_test_cases)
+def test_response_streaming_basic(compat_client, text_model_id, case):
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        stream=True,
+    )
+
+    # Track events and timing to verify proper streaming
+    events = []
+    event_times = []
+    response_id = ""
+
+    start_time = time.time()
+
+    for chunk in response:
+        current_time = time.time()
+        event_times.append(current_time - start_time)
+        events.append(chunk)
+
+        if chunk.type == "response.created":
+            # Verify response.created is emitted first and immediately
+            assert len(events) == 1, "response.created should be the first event"
+            assert event_times[0] < 0.1, "response.created should be emitted immediately"
+            assert chunk.response.status == "in_progress"
+            response_id = chunk.response.id
+
+        elif chunk.type == "response.completed":
+            # Verify response.completed comes after response.created
+            assert len(events) >= 2, "response.completed should come after response.created"
+            assert chunk.response.status == "completed"
+            assert chunk.response.id == response_id, "Response ID should be consistent"
+
+            # Verify content quality
+            output_text = chunk.response.output_text.lower().strip()
+            assert len(output_text) > 0, "Response should have content"
+            assert case.expected.lower() in output_text, f"Expected '{case.expected}' in response"
+
+    # Use validator for common checks
+    validator = StreamingValidator(events)
+    validator.assert_basic_event_sequence()
+    validator.assert_response_consistency()
+
+    # Verify stored response matches streamed response
+    retrieved_response = compat_client.responses.retrieve(response_id=response_id)
+    final_event = events[-1]
+    assert retrieved_response.output_text == final_event.response.output_text
+
+
+@pytest.mark.parametrize("case", basic_test_cases)
+def test_response_streaming_incremental_content(compat_client, text_model_id, case):
+    """Test that streaming actually delivers content incrementally, not just at the end."""
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        stream=True,
+    )
+
+    # Track all events and their content to verify incremental streaming
+    events = []
+    content_snapshots = []
+    event_times = []
+
+    start_time = time.time()
+
+    for chunk in response:
+        current_time = time.time()
+        event_times.append(current_time - start_time)
+        events.append(chunk)
+
+        # Track content at each event based on event type
+        if chunk.type == "response.output_text.delta":
+            # For delta events, track the delta content
+            content_snapshots.append(chunk.delta)
+        elif hasattr(chunk, "response") and hasattr(chunk.response, "output_text"):
+            # For response.created/completed events, track the full output_text
+            content_snapshots.append(chunk.response.output_text)
+        else:
+            content_snapshots.append("")
+
+    validator = StreamingValidator(events)
+    validator.assert_basic_event_sequence()
+
+    # Check if we have incremental content updates
+    event_types = [event.type for event in events]
+    created_index = event_types.index("response.created")
+    completed_index = event_types.index("response.completed")
+
+    # The key test: verify content progression
+    created_content = content_snapshots[created_index]
+    completed_content = content_snapshots[completed_index]
+
+    # Verify that response.created has empty or minimal content
+    assert len(created_content) == 0, f"response.created should have empty content, got: {repr(created_content[:100])}"
+
+    # Verify that response.completed has the full content
+    assert len(completed_content) > 0, "response.completed should have content"
+    assert case.expected.lower() in completed_content.lower(), f"Expected '{case.expected}' in final content"
+
+    # Use validator for incremental content checks
+    delta_content_total = validator.assert_has_incremental_content()
+
+    # Verify that the accumulated delta content matches the final content
+    assert delta_content_total.strip() == completed_content.strip(), (
+        f"Delta content '{delta_content_total}' should match final content '{completed_content}'"
+    )
+
+    # Verify timing: delta events should come between created and completed
+    delta_events = [i for i, event_type in enumerate(event_types) if event_type == "response.output_text.delta"]
+    for delta_idx in delta_events:
+        assert created_index < delta_idx < completed_index, (
+            f"Delta event at index {delta_idx} should be between created ({created_index}) and completed ({completed_index})"
+        )
+
+
+@pytest.mark.parametrize("case", multi_turn_test_cases)
+def test_response_non_streaming_multi_turn(compat_client, text_model_id, case):
+    previous_response_id = None
+    for turn_input, turn_expected in case.turns:
+        response = compat_client.responses.create(
+            model=text_model_id,
+            input=turn_input,
+            previous_response_id=previous_response_id,
+        )
+        previous_response_id = response.id
+        output_text = response.output_text.lower()
+        assert turn_expected.lower() in output_text
+
+
+@pytest.mark.parametrize("case", image_test_cases)
+def test_response_non_streaming_image(compat_client, text_model_id, case):
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        stream=False,
+    )
+    output_text = response.output_text.lower()
+    assert case.expected.lower() in output_text
+
+
+@pytest.mark.parametrize("case", multi_turn_image_test_cases)
+def test_response_non_streaming_multi_turn_image(compat_client, text_model_id, case):
+    previous_response_id = None
+    for turn_input, turn_expected in case.turns:
+        response = compat_client.responses.create(
+            model=text_model_id,
+            input=turn_input,
+            previous_response_id=previous_response_id,
+        )
+        previous_response_id = response.id
+        output_text = response.output_text.lower()
+        assert turn_expected.lower() in output_text
diff --git a/tests/integration/non_ci/responses/test_file_search.py b/tests/integration/non_ci/responses/test_file_search.py
new file mode 100644
index 000000000..ba7775a0b
--- /dev/null
+++ b/tests/integration/non_ci/responses/test_file_search.py
@@ -0,0 +1,318 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import json
+import time
+
+import pytest
+
+from llama_stack import LlamaStackAsLibraryClient
+
+from .helpers import new_vector_store, upload_file
+
+
+@pytest.mark.parametrize(
+    "text_format",
+    # Not testing json_object because most providers don't actually support it.
+    [
+        {"type": "text"},
+        {
+            "type": "json_schema",
+            "name": "capitals",
+            "description": "A schema for the capital of each country",
+            "schema": {"type": "object", "properties": {"capital": {"type": "string"}}},
+            "strict": True,
+        },
+    ],
+)
+def test_response_text_format(compat_client, text_model_id, text_format):
+    if isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("Responses API text format is not yet supported in library client.")
+
+    stream = False
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="What is the capital of France?",
+        stream=stream,
+        text={"format": text_format},
+    )
+    # by_alias=True is needed because otherwise Pydantic renames our "schema" field
+    assert response.text.format.model_dump(exclude_none=True, by_alias=True) == text_format
+    assert "paris" in response.output_text.lower()
+    if text_format["type"] == "json_schema":
+        assert "paris" in json.loads(response.output_text)["capital"].lower()
+
+
+@pytest.fixture
+def vector_store_with_filtered_files(compat_client, text_model_id, tmp_path_factory):
+    """Create a vector store with multiple files that have different attributes for filtering tests."""
+    if isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("Responses API file search is not yet supported in library client.")
+
+    vector_store = new_vector_store(compat_client, "test_vector_store_with_filters")
+    tmp_path = tmp_path_factory.mktemp("filter_test_files")
+
+    # Create multiple files with different attributes
+    files_data = [
+        {
+            "name": "us_marketing_q1.txt",
+            "content": "US promotional campaigns for Q1 2023. Revenue increased by 15% in the US region.",
+            "attributes": {
+                "region": "us",
+                "category": "marketing",
+                "date": 1672531200,  # Jan 1, 2023
+            },
+        },
+        {
+            "name": "us_engineering_q2.txt",
+            "content": "US technical updates for Q2 2023. New features deployed in the US region.",
+            "attributes": {
+                "region": "us",
+                "category": "engineering",
+                "date": 1680307200,  # Apr 1, 2023
+            },
+        },
+        {
+            "name": "eu_marketing_q1.txt",
+            "content": "European advertising campaign results for Q1 2023. Strong growth in EU markets.",
+            "attributes": {
+                "region": "eu",
+                "category": "marketing",
+                "date": 1672531200,  # Jan 1, 2023
+            },
+        },
+        {
+            "name": "asia_sales_q3.txt",
+            "content": "Asia Pacific revenue figures for Q3 2023. Record breaking quarter in Asia.",
+            "attributes": {
+                "region": "asia",
+                "category": "sales",
+                "date": 1688169600,  # Jul 1, 2023
+            },
+        },
+    ]
+
+    file_ids = []
+    for file_data in files_data:
+        # Create file
+        file_path = tmp_path / file_data["name"]
+        file_path.write_text(file_data["content"])
+
+        # Upload file
+        file_response = upload_file(compat_client, file_data["name"], str(file_path))
+        file_ids.append(file_response.id)
+
+        # Attach file to vector store with attributes
+        file_attach_response = compat_client.vector_stores.files.create(
+            vector_store_id=vector_store.id,
+            file_id=file_response.id,
+            attributes=file_data["attributes"],
+        )
+
+        # Wait for attachment
+        while file_attach_response.status == "in_progress":
+            time.sleep(0.1)
+            file_attach_response = compat_client.vector_stores.files.retrieve(
+                vector_store_id=vector_store.id,
+                file_id=file_response.id,
+            )
+        assert file_attach_response.status == "completed"
+
+    yield vector_store
+
+    # Cleanup: delete vector store and files
+    try:
+        compat_client.vector_stores.delete(vector_store_id=vector_store.id)
+        for file_id in file_ids:
+            try:
+                compat_client.files.delete(file_id=file_id)
+            except Exception:
+                pass  # File might already be deleted
+    except Exception:
+        pass  # Best effort cleanup
+
+
+def test_response_file_search_filter_by_region(compat_client, text_model_id, vector_store_with_filtered_files):
+    """Test file search with region equality filter."""
+    tools = [
+        {
+            "type": "file_search",
+            "vector_store_ids": [vector_store_with_filtered_files.id],
+            "filters": {"type": "eq", "key": "region", "value": "us"},
+        }
+    ]
+
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="What are the updates from the US region?",
+        tools=tools,
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    # Verify file search was called with US filter
+    assert len(response.output) > 1
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].results
+    # Should only return US files (not EU or Asia files)
+    for result in response.output[0].results:
+        assert "us" in result.text.lower() or "US" in result.text
+        # Ensure non-US regions are NOT returned
+        assert "european" not in result.text.lower()
+        assert "asia" not in result.text.lower()
+
+
+def test_response_file_search_filter_by_category(compat_client, text_model_id, vector_store_with_filtered_files):
+    """Test file search with category equality filter."""
+    tools = [
+        {
+            "type": "file_search",
+            "vector_store_ids": [vector_store_with_filtered_files.id],
+            "filters": {"type": "eq", "key": "category", "value": "marketing"},
+        }
+    ]
+
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="Show me all marketing reports",
+        tools=tools,
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].results
+    # Should only return marketing files (not engineering or sales)
+    for result in response.output[0].results:
+        # Marketing files should have promotional/advertising content
+        assert "promotional" in result.text.lower() or "advertising" in result.text.lower()
+        # Ensure non-marketing categories are NOT returned
+        assert "technical" not in result.text.lower()
+        assert "revenue figures" not in result.text.lower()
+
+
+def test_response_file_search_filter_by_date_range(compat_client, text_model_id, vector_store_with_filtered_files):
+    """Test file search with date range filter using compound AND."""
+    tools = [
+        {
+            "type": "file_search",
+            "vector_store_ids": [vector_store_with_filtered_files.id],
+            "filters": {
+                "type": "and",
+                "filters": [
+                    {
+                        "type": "gte",
+                        "key": "date",
+                        "value": 1672531200,  # Jan 1, 2023
+                    },
+                    {
+                        "type": "lt",
+                        "key": "date",
+                        "value": 1680307200,  # Apr 1, 2023
+                    },
+                ],
+            },
+        }
+    ]
+
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="What happened in Q1 2023?",
+        tools=tools,
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].results
+    # Should only return Q1 files (not Q2 or Q3)
+    for result in response.output[0].results:
+        assert "q1" in result.text.lower()
+        # Ensure non-Q1 quarters are NOT returned
+        assert "q2" not in result.text.lower()
+        assert "q3" not in result.text.lower()
+
+
+def test_response_file_search_filter_compound_and(compat_client, text_model_id, vector_store_with_filtered_files):
+    """Test file search with compound AND filter (region AND category)."""
+    tools = [
+        {
+            "type": "file_search",
+            "vector_store_ids": [vector_store_with_filtered_files.id],
+            "filters": {
+                "type": "and",
+                "filters": [
+                    {"type": "eq", "key": "region", "value": "us"},
+                    {"type": "eq", "key": "category", "value": "engineering"},
+                ],
+            },
+        }
+    ]
+
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="What are the engineering updates from the US?",
+        tools=tools,
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].results
+    # Should only return US engineering files
+    assert len(response.output[0].results) >= 1
+    for result in response.output[0].results:
+        assert "us" in result.text.lower() and "technical" in result.text.lower()
+        # Ensure it's not from other regions or categories
+        assert "european" not in result.text.lower() and "asia" not in result.text.lower()
+        assert "promotional" not in result.text.lower() and "revenue" not in result.text.lower()
+
+
+def test_response_file_search_filter_compound_or(compat_client, text_model_id, vector_store_with_filtered_files):
+    """Test file search with compound OR filter (marketing OR sales)."""
+    tools = [
+        {
+            "type": "file_search",
+            "vector_store_ids": [vector_store_with_filtered_files.id],
+            "filters": {
+                "type": "or",
+                "filters": [
+                    {"type": "eq", "key": "category", "value": "marketing"},
+                    {"type": "eq", "key": "category", "value": "sales"},
+                ],
+            },
+        }
+    ]
+
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="Show me marketing and sales documents",
+        tools=tools,
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].results
+    # Should return marketing and sales files, but NOT engineering
+    categories_found = set()
+    for result in response.output[0].results:
+        text_lower = result.text.lower()
+        if "promotional" in text_lower or "advertising" in text_lower:
+            categories_found.add("marketing")
+        if "revenue figures" in text_lower:
+            categories_found.add("sales")
+        # Ensure engineering files are NOT returned
+        assert "technical" not in text_lower, f"Engineering file should not be returned, but got: {result.text}"
+
+    # Verify we got at least one of the expected categories
+    assert len(categories_found) > 0, "Should have found at least one marketing or sales file"
+    assert categories_found.issubset({"marketing", "sales"}), f"Found unexpected categories: {categories_found}"
diff --git a/tests/integration/non_ci/responses/test_responses.py b/tests/integration/non_ci/responses/test_responses.py
deleted file mode 100644
index 954f009c2..000000000
--- a/tests/integration/non_ci/responses/test_responses.py
+++ /dev/null
@@ -1,1143 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-import json
-import os
-import time
-
-import httpx
-import openai
-import pytest
-
-from llama_stack import LlamaStackAsLibraryClient
-from llama_stack.core.datatypes import AuthenticationRequiredError
-from tests.common.mcp import dependency_tools, make_mcp_server
-
-from .fixtures.fixtures import case_id_generator
-from .fixtures.load import load_test_cases
-
-responses_test_cases = load_test_cases("responses")
-
-
-def _new_vector_store(openai_client, name):
-    # Ensure we don't reuse an existing vector store
-    vector_stores = openai_client.vector_stores.list()
-    for vector_store in vector_stores:
-        if vector_store.name == name:
-            openai_client.vector_stores.delete(vector_store_id=vector_store.id)
-
-    # Create a new vector store
-    vector_store = openai_client.vector_stores.create(
-        name=name,
-    )
-    return vector_store
-
-
-def _upload_file(openai_client, name, file_path):
-    # Ensure we don't reuse an existing file
-    files = openai_client.files.list()
-    for file in files:
-        if file.filename == name:
-            openai_client.files.delete(file_id=file.id)
-
-    # Upload a text file with our document content
-    return openai_client.files.create(file=open(file_path, "rb"), purpose="assistants")
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_basic"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_basic(request, compat_client, text_model_id, case):
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        stream=False,
-    )
-    output_text = response.output_text.lower().strip()
-    assert len(output_text) > 0
-    assert case["output"].lower() in output_text
-
-    retrieved_response = compat_client.responses.retrieve(response_id=response.id)
-    assert retrieved_response.output_text == response.output_text
-
-    next_response = compat_client.responses.create(
-        model=text_model_id,
-        input="Repeat your previous response in all caps.",
-        previous_response_id=response.id,
-    )
-    next_output_text = next_response.output_text.strip()
-    assert case["output"].upper() in next_output_text
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_basic"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_streaming_basic(request, compat_client, text_model_id, case):
-    import time
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        stream=True,
-    )
-
-    # Track events and timing to verify proper streaming
-    events = []
-    event_times = []
-    response_id = ""
-
-    start_time = time.time()
-
-    for chunk in response:
-        current_time = time.time()
-        event_times.append(current_time - start_time)
-        events.append(chunk)
-
-        if chunk.type == "response.created":
-            # Verify response.created is emitted first and immediately
-            assert len(events) == 1, "response.created should be the first event"
-            assert event_times[0] < 0.1, "response.created should be emitted immediately"
-            assert chunk.response.status == "in_progress"
-            response_id = chunk.response.id
-
-        elif chunk.type == "response.completed":
-            # Verify response.completed comes after response.created
-            assert len(events) >= 2, "response.completed should come after response.created"
-            assert chunk.response.status == "completed"
-            assert chunk.response.id == response_id, "Response ID should be consistent"
-
-            # Verify content quality
-            output_text = chunk.response.output_text.lower().strip()
-            assert len(output_text) > 0, "Response should have content"
-            assert case["output"].lower() in output_text, f"Expected '{case['output']}' in response"
-
-    # Verify we got both required events
-    event_types = [event.type for event in events]
-    assert "response.created" in event_types, "Missing response.created event"
-    assert "response.completed" in event_types, "Missing response.completed event"
-
-    # Verify event order
-    created_index = event_types.index("response.created")
-    completed_index = event_types.index("response.completed")
-    assert created_index < completed_index, "response.created should come before response.completed"
-
-    # Verify stored response matches streamed response
-    retrieved_response = compat_client.responses.retrieve(response_id=response_id)
-    final_event = events[-1]
-    assert retrieved_response.output_text == final_event.response.output_text
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_basic"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_streaming_incremental_content(request, compat_client, text_model_id, case):
-    """Test that streaming actually delivers content incrementally, not just at the end."""
-    import time
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        stream=True,
-    )
-
-    # Track all events and their content to verify incremental streaming
-    events = []
-    content_snapshots = []
-    event_times = []
-
-    start_time = time.time()
-
-    for chunk in response:
-        current_time = time.time()
-        event_times.append(current_time - start_time)
-        events.append(chunk)
-
-        # Track content at each event based on event type
-        if chunk.type == "response.output_text.delta":
-            # For delta events, track the delta content
-            content_snapshots.append(chunk.delta)
-        elif hasattr(chunk, "response") and hasattr(chunk.response, "output_text"):
-            # For response.created/completed events, track the full output_text
-            content_snapshots.append(chunk.response.output_text)
-        else:
-            content_snapshots.append("")
-
-    # Verify we have the expected events
-    event_types = [event.type for event in events]
-    assert "response.created" in event_types, "Missing response.created event"
-    assert "response.completed" in event_types, "Missing response.completed event"
-
-    # Check if we have incremental content updates
-    created_index = event_types.index("response.created")
-    completed_index = event_types.index("response.completed")
-
-    # The key test: verify content progression
-    created_content = content_snapshots[created_index]
-    completed_content = content_snapshots[completed_index]
-
-    # Verify that response.created has empty or minimal content
-    assert len(created_content) == 0, f"response.created should have empty content, got: {repr(created_content[:100])}"
-
-    # Verify that response.completed has the full content
-    assert len(completed_content) > 0, "response.completed should have content"
-    assert case["output"].lower() in completed_content.lower(), f"Expected '{case['output']}' in final content"
-
-    # Check for true incremental streaming by looking for delta events
-    delta_events = [i for i, event_type in enumerate(event_types) if event_type == "response.output_text.delta"]
-
-    # Assert that we have delta events (true incremental streaming)
-    assert len(delta_events) > 0, "Expected delta events for true incremental streaming, but found none"
-
-    # Verify delta events have content and accumulate to final content
-    delta_content_total = ""
-    non_empty_deltas = 0
-
-    for delta_idx in delta_events:
-        delta_content = content_snapshots[delta_idx]
-        if delta_content:
-            delta_content_total += delta_content
-            non_empty_deltas += 1
-
-    # Assert that we have meaningful delta content
-    assert non_empty_deltas > 0, "Delta events found but none contain content"
-    assert len(delta_content_total) > 0, "Delta events found but total delta content is empty"
-
-    # Verify that the accumulated delta content matches the final content
-    assert delta_content_total.strip() == completed_content.strip(), (
-        f"Delta content '{delta_content_total}' should match final content '{completed_content}'"
-    )
-
-    # Verify timing: delta events should come between created and completed
-    for delta_idx in delta_events:
-        assert created_index < delta_idx < completed_index, (
-            f"Delta event at index {delta_idx} should be between created ({created_index}) and completed ({completed_index})"
-        )
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_multi_turn"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_multi_turn(request, compat_client, text_model_id, case):
-    previous_response_id = None
-    for turn in case["turns"]:
-        response = compat_client.responses.create(
-            model=text_model_id,
-            input=turn["input"],
-            previous_response_id=previous_response_id,
-            tools=turn["tools"] if "tools" in turn else None,
-        )
-        previous_response_id = response.id
-        output_text = response.output_text.lower()
-        assert turn["output"].lower() in output_text
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_web_search"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_web_search(request, compat_client, text_model_id, case):
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        tools=case["tools"],
-        stream=False,
-    )
-    assert len(response.output) > 1
-    assert response.output[0].type == "web_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[1].type == "message"
-    assert response.output[1].status == "completed"
-    assert response.output[1].role == "assistant"
-    assert len(response.output[1].content) > 0
-    assert case["output"].lower() in response.output_text.lower().strip()
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_file_search"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_file_search(request, compat_client, text_model_id, tmp_path, case):
-    if isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("Responses API file search is not yet supported in library client.")
-
-    vector_store = _new_vector_store(compat_client, "test_vector_store")
-
-    if "file_content" in case:
-        file_name = "test_response_non_streaming_file_search.txt"
-        file_path = tmp_path / file_name
-        file_path.write_text(case["file_content"])
-    elif "file_path" in case:
-        file_path = os.path.join(os.path.dirname(__file__), "fixtures", case["file_path"])
-        file_name = os.path.basename(file_path)
-    else:
-        raise ValueError(f"No file content or path provided for case {case['case_id']}")
-
-    file_response = _upload_file(compat_client, file_name, file_path)
-
-    # Attach our file to the vector store
-    file_attach_response = compat_client.vector_stores.files.create(
-        vector_store_id=vector_store.id,
-        file_id=file_response.id,
-    )
-
-    # Wait for the file to be attached
-    while file_attach_response.status == "in_progress":
-        time.sleep(0.1)
-        file_attach_response = compat_client.vector_stores.files.retrieve(
-            vector_store_id=vector_store.id,
-            file_id=file_response.id,
-        )
-    assert file_attach_response.status == "completed", f"Expected file to be attached, got {file_attach_response}"
-    assert not file_attach_response.last_error
-
-    # Update our tools with the right vector store id
-    tools = case["tools"]
-    for tool in tools:
-        if tool["type"] == "file_search":
-            tool["vector_store_ids"] = [vector_store.id]
-
-    # Create the response request, which should query our vector store
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        tools=tools,
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    # Verify the file_search_tool was called
-    assert len(response.output) > 1
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].queries  # ensure it's some non-empty list
-    assert response.output[0].results
-    assert case["output"].lower() in response.output[0].results[0].text.lower()
-    assert response.output[0].results[0].score > 0
-
-    # Verify the output_text generated by the response
-    assert case["output"].lower() in response.output_text.lower().strip()
-
-
-def test_response_non_streaming_file_search_empty_vector_store(request, compat_client, text_model_id):
-    if isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("Responses API file search is not yet supported in library client.")
-
-    vector_store = _new_vector_store(compat_client, "test_vector_store")
-
-    # Create the response request, which should query our vector store
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="How many experts does the Llama 4 Maverick model have?",
-        tools=[{"type": "file_search", "vector_store_ids": [vector_store.id]}],
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    # Verify the file_search_tool was called
-    assert len(response.output) > 1
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].queries  # ensure it's some non-empty list
-    assert not response.output[0].results  # ensure we don't get any results
-
-    # Verify some output_text was generated by the response
-    assert response.output_text
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_mcp_tool"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_mcp_tool(request, compat_client, text_model_id, case):
-    if not isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("in-process MCP server is only supported in library client")
-
-    with make_mcp_server() as mcp_server_info:
-        tools = case["tools"]
-        for tool in tools:
-            if tool["type"] == "mcp":
-                tool["server_url"] = mcp_server_info["server_url"]
-
-        response = compat_client.responses.create(
-            model=text_model_id,
-            input=case["input"],
-            tools=tools,
-            stream=False,
-        )
-
-        assert len(response.output) >= 3
-        list_tools = response.output[0]
-        assert list_tools.type == "mcp_list_tools"
-        assert list_tools.server_label == "localmcp"
-        assert len(list_tools.tools) == 2
-        assert {t.name for t in list_tools.tools} == {
-            "get_boiling_point",
-            "greet_everyone",
-        }
-
-        call = response.output[1]
-        assert call.type == "mcp_call"
-        assert call.name == "get_boiling_point"
-        assert json.loads(call.arguments) == {
-            "liquid_name": "myawesomeliquid",
-            "celsius": True,
-        }
-        assert call.error is None
-        assert "-100" in call.output
-
-        # sometimes the model will call the tool again, so we need to get the last message
-        message = response.output[-1]
-        text_content = message.content[0].text
-        assert "boiling point" in text_content.lower()
-
-    with make_mcp_server(required_auth_token="test-token") as mcp_server_info:
-        tools = case["tools"]
-        for tool in tools:
-            if tool["type"] == "mcp":
-                tool["server_url"] = mcp_server_info["server_url"]
-
-        exc_type = (
-            AuthenticationRequiredError
-            if isinstance(compat_client, LlamaStackAsLibraryClient)
-            else (httpx.HTTPStatusError, openai.AuthenticationError)
-        )
-        with pytest.raises(exc_type):
-            compat_client.responses.create(
-                model=text_model_id,
-                input=case["input"],
-                tools=tools,
-                stream=False,
-            )
-
-        for tool in tools:
-            if tool["type"] == "mcp":
-                tool["server_url"] = mcp_server_info["server_url"]
-                tool["headers"] = {"Authorization": "Bearer test-token"}
-
-        response = compat_client.responses.create(
-            model=text_model_id,
-            input=case["input"],
-            tools=tools,
-            stream=False,
-        )
-        assert len(response.output) >= 3
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_custom_tool"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_custom_tool(request, compat_client, text_model_id, case):
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        tools=case["tools"],
-        stream=False,
-    )
-    assert len(response.output) == 1
-    assert response.output[0].type == "function_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].name == "get_weather"
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_image"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_image(request, compat_client, text_model_id, case):
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input=case["input"],
-        stream=False,
-    )
-    output_text = response.output_text.lower()
-    assert case["output"].lower() in output_text
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_multi_turn_image"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_multi_turn_image(request, compat_client, text_model_id, case):
-    previous_response_id = None
-    for turn in case["turns"]:
-        response = compat_client.responses.create(
-            model=text_model_id,
-            input=turn["input"],
-            previous_response_id=previous_response_id,
-            tools=turn["tools"] if "tools" in turn else None,
-        )
-        previous_response_id = response.id
-        output_text = response.output_text.lower()
-        assert turn["output"].lower() in output_text
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_multi_turn_tool_execution"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_non_streaming_multi_turn_tool_execution(compat_client, text_model_id, case):
-    """Test multi-turn tool execution where multiple MCP tool calls are performed in sequence."""
-    if not isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("in-process MCP server is only supported in library client")
-
-    with make_mcp_server(tools=dependency_tools()) as mcp_server_info:
-        tools = case["tools"]
-        # Replace the placeholder URL with the actual server URL
-        for tool in tools:
-            if tool["type"] == "mcp" and tool["server_url"] == "<FILLED_BY_TEST_RUNNER>":
-                tool["server_url"] = mcp_server_info["server_url"]
-
-        response = compat_client.responses.create(
-            input=case["input"],
-            model=text_model_id,
-            tools=tools,
-        )
-
-        # Verify we have MCP tool calls in the output
-        mcp_list_tools = [output for output in response.output if output.type == "mcp_list_tools"]
-
-        mcp_calls = [output for output in response.output if output.type == "mcp_call"]
-        message_outputs = [output for output in response.output if output.type == "message"]
-
-        # Should have exactly 1 MCP list tools message (at the beginning)
-        assert len(mcp_list_tools) == 1, f"Expected exactly 1 mcp_list_tools, got {len(mcp_list_tools)}"
-        assert mcp_list_tools[0].server_label == "localmcp"
-        assert len(mcp_list_tools[0].tools) == 5  # Updated for dependency tools
-        expected_tool_names = {
-            "get_user_id",
-            "get_user_permissions",
-            "check_file_access",
-            "get_experiment_id",
-            "get_experiment_results",
-        }
-        assert {t.name for t in mcp_list_tools[0].tools} == expected_tool_names
-
-        assert len(mcp_calls) >= 1, f"Expected at least 1 mcp_call, got {len(mcp_calls)}"
-        for mcp_call in mcp_calls:
-            assert mcp_call.error is None, f"MCP call should not have errors, got: {mcp_call.error}"
-
-        assert len(message_outputs) >= 1, f"Expected at least 1 message output, got {len(message_outputs)}"
-
-        final_message = message_outputs[-1]
-        assert final_message.role == "assistant", f"Final message should be from assistant, got {final_message.role}"
-        assert final_message.status == "completed", f"Final message should be completed, got {final_message.status}"
-        assert len(final_message.content) > 0, "Final message should have content"
-
-        expected_output = case["output"]
-        assert expected_output.lower() in response.output_text.lower(), (
-            f"Expected '{expected_output}' to appear in response: {response.output_text}"
-        )
-
-
-@pytest.mark.parametrize(
-    "case",
-    responses_test_cases["test_response_multi_turn_tool_execution_streaming"]["test_params"]["case"],
-    ids=case_id_generator,
-)
-def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_id, case):
-    """Test streaming multi-turn tool execution where multiple MCP tool calls are performed in sequence."""
-    if not isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("in-process MCP server is only supported in library client")
-
-    with make_mcp_server(tools=dependency_tools()) as mcp_server_info:
-        tools = case["tools"]
-        # Replace the placeholder URL with the actual server URL
-        for tool in tools:
-            if tool["type"] == "mcp" and tool["server_url"] == "<FILLED_BY_TEST_RUNNER>":
-                tool["server_url"] = mcp_server_info["server_url"]
-
-        stream = compat_client.responses.create(
-            input=case["input"],
-            model=text_model_id,
-            tools=tools,
-            stream=True,
-        )
-
-        chunks = []
-        for chunk in stream:
-            chunks.append(chunk)
-
-        # Should have at least response.created and response.completed
-        assert len(chunks) >= 2, f"Expected at least 2 chunks (created + completed), got {len(chunks)}"
-
-        # First chunk should be response.created
-        assert chunks[0].type == "response.created", f"First chunk should be response.created, got {chunks[0].type}"
-
-        # Last chunk should be response.completed
-        assert chunks[-1].type == "response.completed", (
-            f"Last chunk should be response.completed, got {chunks[-1].type}"
-        )
-
-        # Verify tool call streaming events are present
-        chunk_types = [chunk.type for chunk in chunks]
-
-        # Should have function call or MCP arguments delta/done events for tool calls
-        delta_events = [
-            chunk
-            for chunk in chunks
-            if chunk.type in ["response.function_call_arguments.delta", "response.mcp_call.arguments.delta"]
-        ]
-        done_events = [
-            chunk
-            for chunk in chunks
-            if chunk.type in ["response.function_call_arguments.done", "response.mcp_call.arguments.done"]
-        ]
-
-        # Should have output item events for tool calls
-        item_added_events = [chunk for chunk in chunks if chunk.type == "response.output_item.added"]
-        item_done_events = [chunk for chunk in chunks if chunk.type == "response.output_item.done"]
-
-        # Should have tool execution progress events
-        mcp_in_progress_events = [chunk for chunk in chunks if chunk.type == "response.mcp_call.in_progress"]
-        mcp_completed_events = [chunk for chunk in chunks if chunk.type == "response.mcp_call.completed"]
-
-        # Should have MCP list tools streaming events
-        mcp_list_tools_in_progress_events = [
-            chunk for chunk in chunks if chunk.type == "response.mcp_list_tools.in_progress"
-        ]
-        mcp_list_tools_completed_events = [
-            chunk for chunk in chunks if chunk.type == "response.mcp_list_tools.completed"
-        ]
-
-        # Verify we have substantial streaming activity (not just batch events)
-        assert len(chunks) > 10, f"Expected rich streaming with many events, got only {len(chunks)} chunks"
-
-        # Since this test involves MCP tool calls, we should see streaming events
-        assert len(delta_events) > 0, (
-            f"Expected function_call_arguments.delta or mcp_call.arguments.delta events, got chunk types: {chunk_types}"
-        )
-        assert len(done_events) > 0, (
-            f"Expected function_call_arguments.done or mcp_call.arguments.done events, got chunk types: {chunk_types}"
-        )
-
-        # Should have output item events for function calls
-        assert len(item_added_events) > 0, f"Expected response.output_item.added events, got chunk types: {chunk_types}"
-        assert len(item_done_events) > 0, f"Expected response.output_item.done events, got chunk types: {chunk_types}"
-
-        # Should have tool execution progress events
-        assert len(mcp_in_progress_events) > 0, (
-            f"Expected response.mcp_call.in_progress events, got chunk types: {chunk_types}"
-        )
-        assert len(mcp_completed_events) > 0, (
-            f"Expected response.mcp_call.completed events, got chunk types: {chunk_types}"
-        )
-
-        # Should have MCP list tools streaming events
-        assert len(mcp_list_tools_in_progress_events) > 0, (
-            f"Expected response.mcp_list_tools.in_progress events, got chunk types: {chunk_types}"
-        )
-        assert len(mcp_list_tools_completed_events) > 0, (
-            f"Expected response.mcp_list_tools.completed events, got chunk types: {chunk_types}"
-        )
-        # MCP failed events are optional (only if errors occur)
-
-        # Verify progress events have proper structure
-        for progress_event in mcp_in_progress_events:
-            assert hasattr(progress_event, "item_id"), "Progress event should have 'item_id' field"
-            assert hasattr(progress_event, "output_index"), "Progress event should have 'output_index' field"
-            assert hasattr(progress_event, "sequence_number"), "Progress event should have 'sequence_number' field"
-
-        for completed_event in mcp_completed_events:
-            assert hasattr(completed_event, "sequence_number"), "Completed event should have 'sequence_number' field"
-
-        # Verify MCP list tools events have proper structure
-        for list_tools_progress_event in mcp_list_tools_in_progress_events:
-            assert hasattr(list_tools_progress_event, "sequence_number"), (
-                "MCP list tools progress event should have 'sequence_number' field"
-            )
-
-        for list_tools_completed_event in mcp_list_tools_completed_events:
-            assert hasattr(list_tools_completed_event, "sequence_number"), (
-                "MCP list tools completed event should have 'sequence_number' field"
-            )
-
-        # Verify delta events have proper structure
-        for delta_event in delta_events:
-            assert hasattr(delta_event, "delta"), "Delta event should have 'delta' field"
-            assert hasattr(delta_event, "item_id"), "Delta event should have 'item_id' field"
-            assert hasattr(delta_event, "sequence_number"), "Delta event should have 'sequence_number' field"
-            assert delta_event.delta, "Delta should not be empty"
-
-        # Verify done events have proper structure
-        for done_event in done_events:
-            assert hasattr(done_event, "arguments"), "Done event should have 'arguments' field"
-            assert hasattr(done_event, "item_id"), "Done event should have 'item_id' field"
-            assert done_event.arguments, "Final arguments should not be empty"
-
-        # Verify output item added events have proper structure
-        for added_event in item_added_events:
-            assert hasattr(added_event, "item"), "Added event should have 'item' field"
-            assert hasattr(added_event, "output_index"), "Added event should have 'output_index' field"
-            assert hasattr(added_event, "sequence_number"), "Added event should have 'sequence_number' field"
-            assert hasattr(added_event, "response_id"), "Added event should have 'response_id' field"
-            assert added_event.item.type in ["function_call", "mcp_call", "mcp_list_tools"], (
-                "Added item should be a tool call or MCP list tools"
-            )
-            if added_event.item.type in ["function_call", "mcp_call"]:
-                assert added_event.item.status == "in_progress", "Added tool call should be in progress"
-            # Note: mcp_list_tools doesn't have a status field, it's implicitly completed when added
-            assert added_event.response_id, "Response ID should not be empty"
-            assert isinstance(added_event.output_index, int), "Output index should be integer"
-            assert added_event.output_index >= 0, "Output index should be non-negative"
-
-        # Verify output item done events have proper structure
-        for done_event in item_done_events:
-            assert hasattr(done_event, "item"), "Done event should have 'item' field"
-            assert hasattr(done_event, "output_index"), "Done event should have 'output_index' field"
-            assert hasattr(done_event, "sequence_number"), "Done event should have 'sequence_number' field"
-            assert hasattr(done_event, "response_id"), "Done event should have 'response_id' field"
-            assert done_event.item.type in ["function_call", "mcp_call", "mcp_list_tools"], (
-                "Done item should be a tool call or MCP list tools"
-            )
-            # Note: MCP calls and mcp_list_tools don't have a status field, only function calls do
-            if done_event.item.type == "function_call":
-                assert done_event.item.status == "completed", "Function call should be completed"
-            # Note: mcp_call and mcp_list_tools don't have status fields
-            assert done_event.response_id, "Response ID should not be empty"
-            assert isinstance(done_event.output_index, int), "Output index should be integer"
-            assert done_event.output_index >= 0, "Output index should be non-negative"
-
-        # Group function call and MCP argument events by item_id (these should have proper tracking)
-        argument_events_by_item_id = {}
-        for chunk in chunks:
-            if hasattr(chunk, "item_id") and chunk.type in [
-                "response.function_call_arguments.delta",
-                "response.function_call_arguments.done",
-                "response.mcp_call.arguments.delta",
-                "response.mcp_call.arguments.done",
-            ]:
-                item_id = chunk.item_id
-                if item_id not in argument_events_by_item_id:
-                    argument_events_by_item_id[item_id] = []
-                argument_events_by_item_id[item_id].append(chunk)
-
-        for item_id, related_events in argument_events_by_item_id.items():
-            # Should have at least one delta and one done event for a complete tool call
-            delta_events = [
-                e
-                for e in related_events
-                if e.type in ["response.function_call_arguments.delta", "response.mcp_call.arguments.delta"]
-            ]
-            done_events = [
-                e
-                for e in related_events
-                if e.type in ["response.function_call_arguments.done", "response.mcp_call.arguments.done"]
-            ]
-
-            assert len(delta_events) > 0, f"Item {item_id} should have at least one delta event"
-            assert len(done_events) == 1, f"Item {item_id} should have exactly one done event"
-
-            # Verify all events have the same item_id
-            for event in related_events:
-                assert event.item_id == item_id, f"Event should have consistent item_id {item_id}, got {event.item_id}"
-
-        # Verify content part events if they exist (for text streaming)
-        content_part_added_events = [chunk for chunk in chunks if chunk.type == "response.content_part.added"]
-        content_part_done_events = [chunk for chunk in chunks if chunk.type == "response.content_part.done"]
-
-        # Content part events should be paired (if any exist)
-        if len(content_part_added_events) > 0:
-            assert len(content_part_done_events) > 0, (
-                "Should have content_part.done events if content_part.added events exist"
-            )
-
-            # Verify content part event structure
-            for added_event in content_part_added_events:
-                assert hasattr(added_event, "response_id"), "Content part added event should have response_id"
-                assert hasattr(added_event, "item_id"), "Content part added event should have item_id"
-                assert hasattr(added_event, "part"), "Content part added event should have part"
-
-                # TODO: enable this after the client types are updated
-                # assert added_event.part.type == "output_text", "Content part should be an output_text"
-
-            for done_event in content_part_done_events:
-                assert hasattr(done_event, "response_id"), "Content part done event should have response_id"
-                assert hasattr(done_event, "item_id"), "Content part done event should have item_id"
-                assert hasattr(done_event, "part"), "Content part done event should have part"
-
-                # TODO: enable this after the client types are updated
-                # assert len(done_event.part.text) > 0, "Content part should have text when done"
-
-        # Basic pairing check: each output_item.added should be followed by some activity
-        # (but we can't enforce strict 1:1 pairing due to the complexity of multi-turn scenarios)
-        assert len(item_added_events) > 0, "Should have at least one output_item.added event"
-
-        # Verify response_id consistency across all events
-        response_ids = set()
-        for chunk in chunks:
-            if hasattr(chunk, "response_id"):
-                response_ids.add(chunk.response_id)
-            elif hasattr(chunk, "response") and hasattr(chunk.response, "id"):
-                response_ids.add(chunk.response.id)
-
-        assert len(response_ids) == 1, f"All events should reference the same response_id, found: {response_ids}"
-
-        # Get the final response from the last chunk
-        final_chunk = chunks[-1]
-        if hasattr(final_chunk, "response"):
-            final_response = final_chunk.response
-
-            # Verify multi-turn MCP tool execution results
-            mcp_list_tools = [output for output in final_response.output if output.type == "mcp_list_tools"]
-            mcp_calls = [output for output in final_response.output if output.type == "mcp_call"]
-            message_outputs = [output for output in final_response.output if output.type == "message"]
-
-            # Should have exactly 1 MCP list tools message (at the beginning)
-            assert len(mcp_list_tools) == 1, f"Expected exactly 1 mcp_list_tools, got {len(mcp_list_tools)}"
-            assert mcp_list_tools[0].server_label == "localmcp"
-            assert len(mcp_list_tools[0].tools) == 5  # Updated for dependency tools
-            expected_tool_names = {
-                "get_user_id",
-                "get_user_permissions",
-                "check_file_access",
-                "get_experiment_id",
-                "get_experiment_results",
-            }
-            assert {t.name for t in mcp_list_tools[0].tools} == expected_tool_names
-
-            # Should have at least 1 MCP call (the model should call at least one tool)
-            assert len(mcp_calls) >= 1, f"Expected at least 1 mcp_call, got {len(mcp_calls)}"
-
-            # All MCP calls should be completed (verifies our tool execution works)
-            for mcp_call in mcp_calls:
-                assert mcp_call.error is None, f"MCP call should not have errors, got: {mcp_call.error}"
-
-            # Should have at least one final message response
-            assert len(message_outputs) >= 1, f"Expected at least 1 message output, got {len(message_outputs)}"
-
-            # Final message should be from assistant and completed
-            final_message = message_outputs[-1]
-            assert final_message.role == "assistant", (
-                f"Final message should be from assistant, got {final_message.role}"
-            )
-            assert final_message.status == "completed", f"Final message should be completed, got {final_message.status}"
-            assert len(final_message.content) > 0, "Final message should have content"
-
-            # Check that the expected output appears in the response
-            expected_output = case["output"]
-            assert expected_output.lower() in final_response.output_text.lower(), (
-                f"Expected '{expected_output}' to appear in response: {final_response.output_text}"
-            )
-
-
-@pytest.mark.parametrize(
-    "text_format",
-    # Not testing json_object because most providers don't actually support it.
-    [
-        {"type": "text"},
-        {
-            "type": "json_schema",
-            "name": "capitals",
-            "description": "A schema for the capital of each country",
-            "schema": {"type": "object", "properties": {"capital": {"type": "string"}}},
-            "strict": True,
-        },
-    ],
-)
-def test_response_text_format(compat_client, text_model_id, text_format):
-    if isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("Responses API text format is not yet supported in library client.")
-
-    stream = False
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="What is the capital of France?",
-        stream=stream,
-        text={"format": text_format},
-    )
-    # by_alias=True is needed because otherwise Pydantic renames our "schema" field
-    assert response.text.format.model_dump(exclude_none=True, by_alias=True) == text_format
-    assert "paris" in response.output_text.lower()
-    if text_format["type"] == "json_schema":
-        assert "paris" in json.loads(response.output_text)["capital"].lower()
-
-
-@pytest.fixture
-def vector_store_with_filtered_files(compat_client, text_model_id, tmp_path_factory):
-    """Create a vector store with multiple files that have different attributes for filtering tests."""
-    if isinstance(compat_client, LlamaStackAsLibraryClient):
-        pytest.skip("Responses API file search is not yet supported in library client.")
-
-    vector_store = _new_vector_store(compat_client, "test_vector_store_with_filters")
-    tmp_path = tmp_path_factory.mktemp("filter_test_files")
-
-    # Create multiple files with different attributes
-    files_data = [
-        {
-            "name": "us_marketing_q1.txt",
-            "content": "US promotional campaigns for Q1 2023. Revenue increased by 15% in the US region.",
-            "attributes": {
-                "region": "us",
-                "category": "marketing",
-                "date": 1672531200,  # Jan 1, 2023
-            },
-        },
-        {
-            "name": "us_engineering_q2.txt",
-            "content": "US technical updates for Q2 2023. New features deployed in the US region.",
-            "attributes": {
-                "region": "us",
-                "category": "engineering",
-                "date": 1680307200,  # Apr 1, 2023
-            },
-        },
-        {
-            "name": "eu_marketing_q1.txt",
-            "content": "European advertising campaign results for Q1 2023. Strong growth in EU markets.",
-            "attributes": {
-                "region": "eu",
-                "category": "marketing",
-                "date": 1672531200,  # Jan 1, 2023
-            },
-        },
-        {
-            "name": "asia_sales_q3.txt",
-            "content": "Asia Pacific revenue figures for Q3 2023. Record breaking quarter in Asia.",
-            "attributes": {
-                "region": "asia",
-                "category": "sales",
-                "date": 1688169600,  # Jul 1, 2023
-            },
-        },
-    ]
-
-    file_ids = []
-    for file_data in files_data:
-        # Create file
-        file_path = tmp_path / file_data["name"]
-        file_path.write_text(file_data["content"])
-
-        # Upload file
-        file_response = _upload_file(compat_client, file_data["name"], str(file_path))
-        file_ids.append(file_response.id)
-
-        # Attach file to vector store with attributes
-        file_attach_response = compat_client.vector_stores.files.create(
-            vector_store_id=vector_store.id,
-            file_id=file_response.id,
-            attributes=file_data["attributes"],
-        )
-
-        # Wait for attachment
-        while file_attach_response.status == "in_progress":
-            time.sleep(0.1)
-            file_attach_response = compat_client.vector_stores.files.retrieve(
-                vector_store_id=vector_store.id,
-                file_id=file_response.id,
-            )
-        assert file_attach_response.status == "completed"
-
-    yield vector_store
-
-    # Cleanup: delete vector store and files
-    try:
-        compat_client.vector_stores.delete(vector_store_id=vector_store.id)
-        for file_id in file_ids:
-            try:
-                compat_client.files.delete(file_id=file_id)
-            except Exception:
-                pass  # File might already be deleted
-    except Exception:
-        pass  # Best effort cleanup
-
-
-def test_response_file_search_filter_by_region(compat_client, text_model_id, vector_store_with_filtered_files):
-    """Test file search with region equality filter."""
-    tools = [
-        {
-            "type": "file_search",
-            "vector_store_ids": [vector_store_with_filtered_files.id],
-            "filters": {"type": "eq", "key": "region", "value": "us"},
-        }
-    ]
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="What are the updates from the US region?",
-        tools=tools,
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    # Verify file search was called with US filter
-    assert len(response.output) > 1
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].results
-    # Should only return US files (not EU or Asia files)
-    for result in response.output[0].results:
-        assert "us" in result.text.lower() or "US" in result.text
-        # Ensure non-US regions are NOT returned
-        assert "european" not in result.text.lower()
-        assert "asia" not in result.text.lower()
-
-
-def test_response_file_search_filter_by_category(compat_client, text_model_id, vector_store_with_filtered_files):
-    """Test file search with category equality filter."""
-    tools = [
-        {
-            "type": "file_search",
-            "vector_store_ids": [vector_store_with_filtered_files.id],
-            "filters": {"type": "eq", "key": "category", "value": "marketing"},
-        }
-    ]
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="Show me all marketing reports",
-        tools=tools,
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].results
-    # Should only return marketing files (not engineering or sales)
-    for result in response.output[0].results:
-        # Marketing files should have promotional/advertising content
-        assert "promotional" in result.text.lower() or "advertising" in result.text.lower()
-        # Ensure non-marketing categories are NOT returned
-        assert "technical" not in result.text.lower()
-        assert "revenue figures" not in result.text.lower()
-
-
-def test_response_file_search_filter_by_date_range(compat_client, text_model_id, vector_store_with_filtered_files):
-    """Test file search with date range filter using compound AND."""
-    tools = [
-        {
-            "type": "file_search",
-            "vector_store_ids": [vector_store_with_filtered_files.id],
-            "filters": {
-                "type": "and",
-                "filters": [
-                    {
-                        "type": "gte",
-                        "key": "date",
-                        "value": 1672531200,  # Jan 1, 2023
-                    },
-                    {
-                        "type": "lt",
-                        "key": "date",
-                        "value": 1680307200,  # Apr 1, 2023
-                    },
-                ],
-            },
-        }
-    ]
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="What happened in Q1 2023?",
-        tools=tools,
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].results
-    # Should only return Q1 files (not Q2 or Q3)
-    for result in response.output[0].results:
-        assert "q1" in result.text.lower()
-        # Ensure non-Q1 quarters are NOT returned
-        assert "q2" not in result.text.lower()
-        assert "q3" not in result.text.lower()
-
-
-def test_response_file_search_filter_compound_and(compat_client, text_model_id, vector_store_with_filtered_files):
-    """Test file search with compound AND filter (region AND category)."""
-    tools = [
-        {
-            "type": "file_search",
-            "vector_store_ids": [vector_store_with_filtered_files.id],
-            "filters": {
-                "type": "and",
-                "filters": [
-                    {"type": "eq", "key": "region", "value": "us"},
-                    {"type": "eq", "key": "category", "value": "engineering"},
-                ],
-            },
-        }
-    ]
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="What are the engineering updates from the US?",
-        tools=tools,
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].results
-    # Should only return US engineering files
-    assert len(response.output[0].results) >= 1
-    for result in response.output[0].results:
-        assert "us" in result.text.lower() and "technical" in result.text.lower()
-        # Ensure it's not from other regions or categories
-        assert "european" not in result.text.lower() and "asia" not in result.text.lower()
-        assert "promotional" not in result.text.lower() and "revenue" not in result.text.lower()
-
-
-def test_response_file_search_filter_compound_or(compat_client, text_model_id, vector_store_with_filtered_files):
-    """Test file search with compound OR filter (marketing OR sales)."""
-    tools = [
-        {
-            "type": "file_search",
-            "vector_store_ids": [vector_store_with_filtered_files.id],
-            "filters": {
-                "type": "or",
-                "filters": [
-                    {"type": "eq", "key": "category", "value": "marketing"},
-                    {"type": "eq", "key": "category", "value": "sales"},
-                ],
-            },
-        }
-    ]
-
-    response = compat_client.responses.create(
-        model=text_model_id,
-        input="Show me marketing and sales documents",
-        tools=tools,
-        stream=False,
-        include=["file_search_call.results"],
-    )
-
-    assert response.output[0].type == "file_search_call"
-    assert response.output[0].status == "completed"
-    assert response.output[0].results
-    # Should return marketing and sales files, but NOT engineering
-    categories_found = set()
-    for result in response.output[0].results:
-        text_lower = result.text.lower()
-        if "promotional" in text_lower or "advertising" in text_lower:
-            categories_found.add("marketing")
-        if "revenue figures" in text_lower:
-            categories_found.add("sales")
-        # Ensure engineering files are NOT returned
-        assert "technical" not in text_lower, f"Engineering file should not be returned, but got: {result.text}"
-
-    # Verify we got at least one of the expected categories
-    assert len(categories_found) > 0, "Should have found at least one marketing or sales file"
-    assert categories_found.issubset({"marketing", "sales"}), f"Found unexpected categories: {categories_found}"
diff --git a/tests/integration/non_ci/responses/test_tool_responses.py b/tests/integration/non_ci/responses/test_tool_responses.py
new file mode 100644
index 000000000..33d109863
--- /dev/null
+++ b/tests/integration/non_ci/responses/test_tool_responses.py
@@ -0,0 +1,335 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import json
+import os
+
+import httpx
+import openai
+import pytest
+from fixtures.test_cases import (
+    custom_tool_test_cases,
+    file_search_test_cases,
+    mcp_tool_test_cases,
+    multi_turn_tool_execution_streaming_test_cases,
+    multi_turn_tool_execution_test_cases,
+    web_search_test_cases,
+)
+from helpers import new_vector_store, setup_mcp_tools, upload_file, wait_for_file_attachment
+from streaming_assertions import StreamingValidator
+
+from llama_stack import LlamaStackAsLibraryClient
+from llama_stack.core.datatypes import AuthenticationRequiredError
+from tests.common.mcp import dependency_tools, make_mcp_server
+
+
+@pytest.mark.parametrize("case", web_search_test_cases)
+def test_response_non_streaming_web_search(compat_client, text_model_id, case):
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        tools=case.tools,
+        stream=False,
+    )
+    assert len(response.output) > 1
+    assert response.output[0].type == "web_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[1].type == "message"
+    assert response.output[1].status == "completed"
+    assert response.output[1].role == "assistant"
+    assert len(response.output[1].content) > 0
+    assert case.expected.lower() in response.output_text.lower().strip()
+
+
+@pytest.mark.parametrize("case", file_search_test_cases)
+def test_response_non_streaming_file_search(compat_client, text_model_id, tmp_path, case):
+    if isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("Responses API file search is not yet supported in library client.")
+
+    vector_store = new_vector_store(compat_client, "test_vector_store")
+
+    if case.file_content:
+        file_name = "test_response_non_streaming_file_search.txt"
+        file_path = tmp_path / file_name
+        file_path.write_text(case.file_content)
+    elif case.file_path:
+        file_path = os.path.join(os.path.dirname(__file__), "fixtures", case.file_path)
+        file_name = os.path.basename(file_path)
+    else:
+        raise ValueError("No file content or path provided for case")
+
+    file_response = upload_file(compat_client, file_name, file_path)
+
+    # Attach our file to the vector store
+    compat_client.vector_stores.files.create(
+        vector_store_id=vector_store.id,
+        file_id=file_response.id,
+    )
+
+    # Wait for the file to be attached
+    wait_for_file_attachment(compat_client, vector_store.id, file_response.id)
+
+    # Update our tools with the right vector store id
+    tools = case.tools
+    for tool in tools:
+        if tool["type"] == "file_search":
+            tool["vector_store_ids"] = [vector_store.id]
+
+    # Create the response request, which should query our vector store
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        tools=tools,
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    # Verify the file_search_tool was called
+    assert len(response.output) > 1
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].queries  # ensure it's some non-empty list
+    assert response.output[0].results
+    assert case.expected.lower() in response.output[0].results[0].text.lower()
+    assert response.output[0].results[0].score > 0
+
+    # Verify the output_text generated by the response
+    assert case.expected.lower() in response.output_text.lower().strip()
+
+
+def test_response_non_streaming_file_search_empty_vector_store(compat_client, text_model_id):
+    if isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("Responses API file search is not yet supported in library client.")
+
+    vector_store = new_vector_store(compat_client, "test_vector_store")
+
+    # Create the response request, which should query our vector store
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input="How many experts does the Llama 4 Maverick model have?",
+        tools=[{"type": "file_search", "vector_store_ids": [vector_store.id]}],
+        stream=False,
+        include=["file_search_call.results"],
+    )
+
+    # Verify the file_search_tool was called
+    assert len(response.output) > 1
+    assert response.output[0].type == "file_search_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].queries  # ensure it's some non-empty list
+    assert not response.output[0].results  # ensure we don't get any results
+
+    # Verify some output_text was generated by the response
+    assert response.output_text
+
+
+@pytest.mark.parametrize("case", mcp_tool_test_cases)
+def test_response_non_streaming_mcp_tool(compat_client, text_model_id, case):
+    if not isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("in-process MCP server is only supported in library client")
+
+    with make_mcp_server() as mcp_server_info:
+        tools = setup_mcp_tools(case.tools, mcp_server_info)
+
+        response = compat_client.responses.create(
+            model=text_model_id,
+            input=case.input,
+            tools=tools,
+            stream=False,
+        )
+
+        assert len(response.output) >= 3
+        list_tools = response.output[0]
+        assert list_tools.type == "mcp_list_tools"
+        assert list_tools.server_label == "localmcp"
+        assert len(list_tools.tools) == 2
+        assert {t.name for t in list_tools.tools} == {
+            "get_boiling_point",
+            "greet_everyone",
+        }
+
+        call = response.output[1]
+        assert call.type == "mcp_call"
+        assert call.name == "get_boiling_point"
+        assert json.loads(call.arguments) == {
+            "liquid_name": "myawesomeliquid",
+            "celsius": True,
+        }
+        assert call.error is None
+        assert "-100" in call.output
+
+        # sometimes the model will call the tool again, so we need to get the last message
+        message = response.output[-1]
+        text_content = message.content[0].text
+        assert "boiling point" in text_content.lower()
+
+    with make_mcp_server(required_auth_token="test-token") as mcp_server_info:
+        tools = setup_mcp_tools(case.tools, mcp_server_info)
+
+        exc_type = (
+            AuthenticationRequiredError
+            if isinstance(compat_client, LlamaStackAsLibraryClient)
+            else (httpx.HTTPStatusError, openai.AuthenticationError)
+        )
+        with pytest.raises(exc_type):
+            compat_client.responses.create(
+                model=text_model_id,
+                input=case.input,
+                tools=tools,
+                stream=False,
+            )
+
+        for tool in tools:
+            if tool["type"] == "mcp":
+                tool["headers"] = {"Authorization": "Bearer test-token"}
+
+        response = compat_client.responses.create(
+            model=text_model_id,
+            input=case.input,
+            tools=tools,
+            stream=False,
+        )
+        assert len(response.output) >= 3
+
+
+@pytest.mark.parametrize("case", custom_tool_test_cases)
+def test_response_non_streaming_custom_tool(compat_client, text_model_id, case):
+    response = compat_client.responses.create(
+        model=text_model_id,
+        input=case.input,
+        tools=case.tools,
+        stream=False,
+    )
+    assert len(response.output) == 1
+    assert response.output[0].type == "function_call"
+    assert response.output[0].status == "completed"
+    assert response.output[0].name == "get_weather"
+
+
+@pytest.mark.parametrize("case", multi_turn_tool_execution_test_cases)
+def test_response_non_streaming_multi_turn_tool_execution(compat_client, text_model_id, case):
+    """Test multi-turn tool execution where multiple MCP tool calls are performed in sequence."""
+    if not isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("in-process MCP server is only supported in library client")
+
+    with make_mcp_server(tools=dependency_tools()) as mcp_server_info:
+        tools = setup_mcp_tools(case.tools, mcp_server_info)
+
+        response = compat_client.responses.create(
+            input=case.input,
+            model=text_model_id,
+            tools=tools,
+        )
+
+        # Verify we have MCP tool calls in the output
+        mcp_list_tools = [output for output in response.output if output.type == "mcp_list_tools"]
+        mcp_calls = [output for output in response.output if output.type == "mcp_call"]
+        message_outputs = [output for output in response.output if output.type == "message"]
+
+        # Should have exactly 1 MCP list tools message (at the beginning)
+        assert len(mcp_list_tools) == 1, f"Expected exactly 1 mcp_list_tools, got {len(mcp_list_tools)}"
+        assert mcp_list_tools[0].server_label == "localmcp"
+        assert len(mcp_list_tools[0].tools) == 5  # Updated for dependency tools
+        expected_tool_names = {
+            "get_user_id",
+            "get_user_permissions",
+            "check_file_access",
+            "get_experiment_id",
+            "get_experiment_results",
+        }
+        assert {t.name for t in mcp_list_tools[0].tools} == expected_tool_names
+
+        assert len(mcp_calls) >= 1, f"Expected at least 1 mcp_call, got {len(mcp_calls)}"
+        for mcp_call in mcp_calls:
+            assert mcp_call.error is None, f"MCP call should not have errors, got: {mcp_call.error}"
+
+        assert len(message_outputs) >= 1, f"Expected at least 1 message output, got {len(message_outputs)}"
+
+        final_message = message_outputs[-1]
+        assert final_message.role == "assistant", f"Final message should be from assistant, got {final_message.role}"
+        assert final_message.status == "completed", f"Final message should be completed, got {final_message.status}"
+        assert len(final_message.content) > 0, "Final message should have content"
+
+        expected_output = case.expected
+        assert expected_output.lower() in response.output_text.lower(), (
+            f"Expected '{expected_output}' to appear in response: {response.output_text}"
+        )
+
+
+@pytest.mark.parametrize("case", multi_turn_tool_execution_streaming_test_cases)
+def test_response_streaming_multi_turn_tool_execution(compat_client, text_model_id, case):
+    """Test streaming multi-turn tool execution where multiple MCP tool calls are performed in sequence."""
+    if not isinstance(compat_client, LlamaStackAsLibraryClient):
+        pytest.skip("in-process MCP server is only supported in library client")
+
+    with make_mcp_server(tools=dependency_tools()) as mcp_server_info:
+        tools = setup_mcp_tools(case.tools, mcp_server_info)
+
+        stream = compat_client.responses.create(
+            input=case.input,
+            model=text_model_id,
+            tools=tools,
+            stream=True,
+        )
+
+        chunks = []
+        for chunk in stream:
+            chunks.append(chunk)
+
+        # Use validator for common streaming checks
+        validator = StreamingValidator(chunks)
+        validator.assert_basic_event_sequence()
+        validator.assert_response_consistency()
+        validator.assert_has_tool_calls()
+        validator.assert_has_mcp_events()
+        validator.assert_rich_streaming()
+
+        # Get the final response from the last chunk
+        final_chunk = chunks[-1]
+        if hasattr(final_chunk, "response"):
+            final_response = final_chunk.response
+
+            # Verify multi-turn MCP tool execution results
+            mcp_list_tools = [output for output in final_response.output if output.type == "mcp_list_tools"]
+            mcp_calls = [output for output in final_response.output if output.type == "mcp_call"]
+            message_outputs = [output for output in final_response.output if output.type == "message"]
+
+            # Should have exactly 1 MCP list tools message (at the beginning)
+            assert len(mcp_list_tools) == 1, f"Expected exactly 1 mcp_list_tools, got {len(mcp_list_tools)}"
+            assert mcp_list_tools[0].server_label == "localmcp"
+            assert len(mcp_list_tools[0].tools) == 5  # Updated for dependency tools
+            expected_tool_names = {
+                "get_user_id",
+                "get_user_permissions",
+                "check_file_access",
+                "get_experiment_id",
+                "get_experiment_results",
+            }
+            assert {t.name for t in mcp_list_tools[0].tools} == expected_tool_names
+
+            # Should have at least 1 MCP call (the model should call at least one tool)
+            assert len(mcp_calls) >= 1, f"Expected at least 1 mcp_call, got {len(mcp_calls)}"
+
+            # All MCP calls should be completed (verifies our tool execution works)
+            for mcp_call in mcp_calls:
+                assert mcp_call.error is None, f"MCP call should not have errors, got: {mcp_call.error}"
+
+            # Should have at least one final message response
+            assert len(message_outputs) >= 1, f"Expected at least 1 message output, got {len(message_outputs)}"
+
+            # Final message should be from assistant and completed
+            final_message = message_outputs[-1]
+            assert final_message.role == "assistant", (
+                f"Final message should be from assistant, got {final_message.role}"
+            )
+            assert final_message.status == "completed", f"Final message should be completed, got {final_message.status}"
+            assert len(final_message.content) > 0, "Final message should have content"
+
+            # Check that the expected output appears in the response
+            expected_output = case.expected
+            assert expected_output.lower() in final_response.output_text.lower(), (
+                f"Expected '{expected_output}' to appear in response: {final_response.output_text}"
+            )

From 01b2afd4b571e580675c15083c687f0d3a8988aa Mon Sep 17 00:00:00 2001
From: ashwinb <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 03:52:45 +0000
Subject: [PATCH 34/45] fix(tests): record missing tests for
 test_responses_store (#3163)

# What does this PR do?

Updates test recordings.

## Test Plan

Started ollama serving the 3.2:3b model. Then ran the server:

```
LLAMA_STACK_TEST_INFERENCE_MODE=record \
  LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings/ \
  SQLITE_STORE_DIR=$(mktemp -d) \
  OLLAMA_URL=http://localhost:11434 \
  llama stack build --template starter --image-type venv --run
```

Then ran the tests which needed recording:

```
pytest -sv tests/integration/agents/test_openai_responses.py \
  --stack-config=server:starter \
   --text-model ollama/llama3.2:3b-instruct-fp16 -k test_responses_store
```

Then, restarted the server with `LLAMA_STACK_TEST_INFERENCE_MODE=replay`, re-ran the tests and verified they passed.
---
 tests/integration/recordings/index.sqlite     |  Bin 53248 -> 53248 bytes
 .../recordings/responses/4a3a4447b16b.json    |    2 +-
 .../recordings/responses/6fe1d4fedf12.json    | 7666 +++++++++--------
 .../recordings/responses/d0ac68cbde69.json    |   21 +-
 .../recordings/responses/decfd950646c.json    |  109 +
 5 files changed, 4282 insertions(+), 3516 deletions(-)
 create mode 100644 tests/integration/recordings/responses/decfd950646c.json

diff --git a/tests/integration/recordings/index.sqlite b/tests/integration/recordings/index.sqlite
index 6f73bb3a00053c8d4182c9412d59346c831d6f04..1951ee7d61338c23da0e6a96cd6a585eb0adf134 100644
GIT binary patch
delta 476
zcmZozz}&Ead4e>f#Y7orK??@GvTk1f9}G-<oeX@P{D(IdO7Ly&;JX;gX5_}w$jda@
z!Odav{1gvH?aePzdIXqQO()OSbltqW)KiO5VlwLjElyVEe5Okb*BBf&TP!ePWMP@j
zn6|MniE;C#MJE|q7#JAlPAr_c*<$G#E@o+ln91xL6vbGUvv4uzGF@OYW}L>B#CD6j
zfZ-=Y%)~;uh(^gSbB;z{eN%g5&Xm;Tv=mEI12Yq|WV0lrMDt`zQxik;q?9y6!!$D!
z!&I|GQ^REQq_o6j%M|lephRk#MPhPFVw!Q1i7}9Yq<`{;P%C~zQw0NKD`PV&LnEL)
zKz}Uh<3@-XY@W9991ElN<Z6Ko0SjG-gb`4&L1~xK<h$Flp!z{VhK)iUu9N2owN9SD
zlV|hu?cL0r#vo@)^BQwZwm<7IdG~Gwn9-(ICYGC@?UrL<R$y0{%zi+TiCtl{#({5a
dllk*losb<#jwe~oH>V$$WMeem+<3O02>?C(k=_6R

delta 288
zcmZozz}&Ead4e>f%0wAwK@|qQE<s-Y9}G-<oeX@P{D(IdO7Ly&;JX;gChNx1$jda@
z!Odav{1gvHsm(7^dIXqQL?+MIbltqW)KiO5ax&`zEza32tjzgLml&>Xwpd`o$il$D
zn6|MniE;C#MJE|q8YMeKCjZ^gH+jMmE=JSM7E8}?F-tN|o6NpJQHZ;MEs5<G%W@Vj
z=3J%=Ova4UHWtP(Zl1RB91Ek=<Z6Ko0SjFNGX+CSD^p`DQ}fa;*~xdeW$_~kH41gO
zPM#ywI(hz1p3Tp<cQbPeb2RemOY<6YOtwGkFnRZG1#X0Lv(3+T%ds%av!_pHKcKi-
c>%cd*i4C5U6^^rR&N(i^#wfhG@oYU404zRTfB*mh

diff --git a/tests/integration/recordings/responses/4a3a4447b16b.json b/tests/integration/recordings/responses/4a3a4447b16b.json
index dbaec07e9..96b40a792 100644
--- a/tests/integration/recordings/responses/4a3a4447b16b.json
+++ b/tests/integration/recordings/responses/4a3a4447b16b.json
@@ -14,7 +14,7 @@
         "models": [
           {
             "model": "nomic-embed-text:latest",
-            "modified_at": "2025-08-05T14:04:07.946926-07:00",
+            "modified_at": "2025-08-14T20:26:10.795125-07:00",
             "digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f",
             "size": 274302450,
             "details": {
diff --git a/tests/integration/recordings/responses/6fe1d4fedf12.json b/tests/integration/recordings/responses/6fe1d4fedf12.json
index 4db74b4e9..733c7bd55 100644
--- a/tests/integration/recordings/responses/6fe1d4fedf12.json
+++ b/tests/integration/recordings/responses/6fe1d4fedf12.json
@@ -1,7 +1,7 @@
 {
   "request": {
     "method": "POST",
-    "url": "http://0.0.0.0:11434/v1/v1/chat/completions",
+    "url": "http://localhost:11434/v1/v1/chat/completions",
     "headers": {},
     "body": {
       "model": "llama3.2:3b-instruct-fp16",
@@ -24,7 +24,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -39,7 +39,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090066,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -50,7 +50,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -65,7 +65,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090066,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -76,7 +76,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -91,7 +91,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090066,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -102,7 +102,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -117,7 +117,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090066,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -128,7 +128,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -143,7 +143,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090066,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -154,7 +154,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -169,7 +169,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090066,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -180,7 +180,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -195,7 +195,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228961,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -206,7 +206,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -221,7 +221,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -232,7 +232,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -247,7 +247,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -258,7 +258,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -273,7 +273,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -284,7 +284,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -299,7 +299,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -310,7 +310,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -325,7 +325,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -336,7 +336,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -351,7 +351,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -362,7 +362,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -377,7 +377,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -388,7 +388,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -403,7 +403,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -414,7 +414,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -429,7 +429,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -440,7 +440,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -455,7 +455,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -466,11 +466,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " general",
+                "content": " you",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -481,7 +481,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -492,7 +492,33 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " with",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228962,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -507,7 +533,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -518,11 +544,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " about",
+                "content": " on",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -533,7 +559,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090067,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -544,3335 +570,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Tokyo",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090067,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "'s",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090067,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " climate",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090067,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ".\n\n",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090067,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "Tok",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090067,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "yo",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090067,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " has",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " a",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " humid",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " subt",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "ropical",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " climate",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " with",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " distinct",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " seasons",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ":\n\n",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "*",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Spring",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "March",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " to",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " May",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "):",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Mild",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " temperatures",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ",",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090068,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " ranging",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " from",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " ",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "10",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "20",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0C",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "50",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "68",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0F",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "),",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " with",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " occasional",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " rain",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " showers",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ".\n",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "*",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090069,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Summer",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "June",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " to",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " August",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "):",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Hot",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " and",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " humid",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ",",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " with",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " average",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " high",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " temperatures",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " around",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " ",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "28",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "30",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0C",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090070,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "82",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "86",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0F",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ").\n",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "*",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Autumn",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "September",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " to",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " November",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "):",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Comfort",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "able",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " temperatures",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ",",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " ranging",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " from",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090071,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " ",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "10",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "25",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0C",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "50",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "77",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0F",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "),",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " with",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " gentle",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " rainfall",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ".\n",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "*",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Winter",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "December",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " to",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090072,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " February",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "):",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " Cool",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " temperatures",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ",",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " averaging",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " around",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " ",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "0",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "10",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0C",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " (",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "32",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "-",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "50",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "\u00b0F",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "),",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " with",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090073,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " occasional",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090074,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " cold",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090074,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " snaps",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090074,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": ".\n\n",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090074,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": "For",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090074,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -3887,7 +585,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -3898,11 +596,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " most",
+                "content": " typical",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -3913,7 +611,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -3924,11 +622,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " up",
+                "content": " climate",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -3939,7 +637,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -3950,11 +648,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": "-to",
+                "content": " of",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -3965,7 +663,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -3976,11 +674,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": "-date",
+                "content": " Tokyo",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -3991,7 +689,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228962,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4002,7 +700,267 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " or",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228962,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " suggest",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228962,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " ways",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228962,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " for",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228962,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " you",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228962,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228963,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " find",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228963,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " out",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228963,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " the",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228963,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " current",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228963,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -4017,7 +975,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4028,11 +986,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " information",
+                "content": ".\n\n",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4043,7 +1001,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4054,11 +1012,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": ",",
+                "content": "Tok",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4069,7 +1027,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4080,11 +1038,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " I",
+                "content": "yo",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4095,7 +1053,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4106,11 +1064,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " recommend",
+                "content": " has",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4121,7 +1079,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4132,33 +1090,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " checking",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090074,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -4173,7 +1105,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4184,11 +1116,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " reliable",
+                "content": " humid",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4199,7 +1131,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4210,11 +1142,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " weather",
+                "content": " subt",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4225,7 +1157,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4236,11 +1168,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " website",
+                "content": "ropical",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4251,7 +1183,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090074,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4262,11 +1194,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " or",
+                "content": " climate",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4277,7 +1209,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4288,33 +1220,7 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
-          "choices": [
-            {
-              "delta": {
-                "content": " app",
-                "function_call": null,
-                "refusal": null,
-                "role": "assistant",
-                "tool_calls": null
-              },
-              "finish_reason": null,
-              "index": 0,
-              "logprobs": null
-            }
-          ],
-          "created": 1754090075,
-          "model": "llama3.2:3b-instruct-fp16",
-          "object": "chat.completion.chunk",
-          "service_tier": null,
-          "system_fingerprint": "fp_ollama",
-          "usage": null
-        }
-      },
-      {
-        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
-        "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -4329,7 +1235,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4340,11 +1246,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " such",
+                "content": " characterized",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4355,7 +1261,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4366,11 +1272,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " as",
+                "content": " by",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4381,7 +1287,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4392,11 +1298,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " Acc",
+                "content": " hot",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4407,7 +1313,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4418,11 +1324,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": "u",
+                "content": " and",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4433,7 +1339,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4444,11 +1350,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": "Weather",
+                "content": " humid",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4459,7 +1365,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4470,11 +1376,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " or",
+                "content": " summers",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4485,7 +1391,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4496,11 +1402,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": " Weather",
+                "content": ",",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4511,7 +1417,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4522,11 +1428,11 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
-                "content": ".com",
+                "content": " mild",
                 "function_call": null,
                 "refusal": null,
                 "role": "assistant",
@@ -4537,7 +1443,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228963,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4548,7 +1454,215 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " winters",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ",",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " and",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " moderate",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " spring",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " and",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " autumn",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " seasons",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -4563,7 +1677,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228964,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
@@ -4574,7 +1688,3569 @@
       {
         "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
         "__data__": {
-          "id": "chatcmpl-751",
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Here",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "'s",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " a",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " general",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " idea",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " of",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " what",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " you",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " might",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " expect",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ":\n\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "*",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Summer",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "June",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228964,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " August",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "):",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Hot",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " and",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " humid",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ",",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " with",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " temperatures",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " often",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " reaching",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " ",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "30",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\u00b0C",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "86",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\u00b0F",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ")",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " or",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " higher",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ".\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "*",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Autumn",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "September",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228965,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " November",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "):",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Mild",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ",",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " with",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " temperatures",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " ranging",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " from",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " ",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "10",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\u00b0C",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "50",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\u00b0F",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ")",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " ",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "20",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\u00b0C",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "68",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\u00b0F",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ").\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228966,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "*",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Spring",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "March",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " May",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ")",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " and",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Winter",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " (",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "December",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " February",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "):",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Cool",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " and",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " sometimes",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " rainy",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ".\n\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "If",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " you",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " need",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " up",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "-to",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228967,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "-date",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " information",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " on",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " the",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " current",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " weather",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " in",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Tokyo",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ",",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " I",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " recommend",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " checking",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " a",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " reliable",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " online",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " weather",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " source",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " such",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " as",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ":\n\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "-",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Acc",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "u",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "Weather",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228968,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "-",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " BBC",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Weather",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "-",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " The",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Weather",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " Channel",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "\n\n",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": "Or",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " you",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " can",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " check",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " local",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " news",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " websites",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " or",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " mobile",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " apps",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " for",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " the",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " latest",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": " forecast",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228969,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
+          "choices": [
+            {
+              "delta": {
+                "content": ".",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228970,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-381",
           "choices": [
             {
               "delta": {
@@ -4589,7 +5265,7 @@
               "logprobs": null
             }
           ],
-          "created": 1754090075,
+          "created": 1755228970,
           "model": "llama3.2:3b-instruct-fp16",
           "object": "chat.completion.chunk",
           "service_tier": null,
diff --git a/tests/integration/recordings/responses/d0ac68cbde69.json b/tests/integration/recordings/responses/d0ac68cbde69.json
index e9a939aa0..b37962fb6 100644
--- a/tests/integration/recordings/responses/d0ac68cbde69.json
+++ b/tests/integration/recordings/responses/d0ac68cbde69.json
@@ -11,26 +11,7 @@
     "body": {
       "__type__": "ollama._types.ProcessResponse",
       "__data__": {
-        "models": [
-          {
-            "model": "llama3.2:3b",
-            "name": "llama3.2:3b",
-            "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
-            "expires_at": "2025-08-06T15:57:21.573326-04:00",
-            "size": 4030033920,
-            "size_vram": 4030033920,
-            "details": {
-              "parent_model": "",
-              "format": "gguf",
-              "family": "llama",
-              "families": [
-                "llama"
-              ],
-              "parameter_size": "3.2B",
-              "quantization_level": "Q4_K_M"
-            }
-          }
-        ]
+        "models": []
       }
     },
     "is_streaming": false
diff --git a/tests/integration/recordings/responses/decfd950646c.json b/tests/integration/recordings/responses/decfd950646c.json
new file mode 100644
index 000000000..f62340c27
--- /dev/null
+++ b/tests/integration/recordings/responses/decfd950646c.json
@@ -0,0 +1,109 @@
+{
+  "request": {
+    "method": "POST",
+    "url": "http://localhost:11434/v1/v1/chat/completions",
+    "headers": {},
+    "body": {
+      "model": "llama3.2:3b-instruct-fp16",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What's the weather in Tokyo? YOU MUST USE THE get_weather function to get the weather."
+        }
+      ],
+      "response_format": {
+        "type": "text"
+      },
+      "stream": true,
+      "tools": [
+        {
+          "type": "function",
+          "function": {
+            "type": "function",
+            "name": "get_weather",
+            "description": "Get the weather in a given city",
+            "parameters": {
+              "type": "object",
+              "properties": {
+                "city": {
+                  "type": "string",
+                  "description": "The city to get the weather for"
+                }
+              }
+            },
+            "strict": null
+          }
+        }
+      ]
+    },
+    "endpoint": "/v1/chat/completions",
+    "model": "llama3.2:3b-instruct-fp16"
+  },
+  "response": {
+    "body": [
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-620",
+          "choices": [
+            {
+              "delta": {
+                "content": "",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": [
+                  {
+                    "index": 0,
+                    "id": "call_490d5ur7",
+                    "function": {
+                      "arguments": "{\"city\":\"Tokyo\"}",
+                      "name": "get_weather"
+                    },
+                    "type": "function"
+                  }
+                ]
+              },
+              "finish_reason": null,
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228972,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      },
+      {
+        "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
+        "__data__": {
+          "id": "chatcmpl-620",
+          "choices": [
+            {
+              "delta": {
+                "content": "",
+                "function_call": null,
+                "refusal": null,
+                "role": "assistant",
+                "tool_calls": null
+              },
+              "finish_reason": "tool_calls",
+              "index": 0,
+              "logprobs": null
+            }
+          ],
+          "created": 1755228972,
+          "model": "llama3.2:3b-instruct-fp16",
+          "object": "chat.completion.chunk",
+          "service_tier": null,
+          "system_fingerprint": "fp_ollama",
+          "usage": null
+        }
+      }
+    ],
+    "is_streaming": true
+  }
+}

From 81ecaf62218898f6911112f80cf2da611d58bad3 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Thu, 14 Aug 2025 21:06:08 -0700
Subject: [PATCH 35/45] fix(ci): make the Vector IO CI follow the same pattern
 as others (#3164)

# What does this PR do?
Updates the integration-vector-io-tests workflow to run daily tests on
Python 3.13 while limiting regular PR tests to Python 3.12 only.

The PR also improves the concurrency configuration to prevent workflow
conflicts between main branch runs and PR runs.

## Test Plan


[![testinprod](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/WjlTemxb6oA4PgZFmj08/2645295d-f421-49ae-8f3f-f4672d8204e2/testinprod.jpeg)](https://app.graphite.dev/settings/meme-library?org=llamastack)
---
 .github/workflows/integration-vector-io-tests.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-vector-io-tests.yml b/.github/workflows/integration-vector-io-tests.yml
index f4d28e407..99a44c147 100644
--- a/.github/workflows/integration-vector-io-tests.yml
+++ b/.github/workflows/integration-vector-io-tests.yml
@@ -14,9 +14,11 @@ on:
       - 'pyproject.toml'
       - 'requirements.txt'
       - '.github/workflows/integration-vector-io-tests.yml' # This workflow
+  schedule:
+    - cron: '0 0 * * *'  # (test on python 3.13) Daily at 12 AM UTC
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -25,7 +27,7 @@ jobs:
     strategy:
       matrix:
         vector-io-provider: ["inline::faiss", "inline::sqlite-vec", "inline::milvus", "remote::chromadb", "remote::pgvector", "remote::weaviate", "remote::qdrant"]
-        python-version: ["3.12", "3.13"]
+        python-version: ${{ github.event.schedule == '0 0 * * *' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }}
       fail-fast: false # we want to run all tests regardless of failure
 
     steps:

From f66ae3b3b152676f66bf7d7132cb9ba26abbee9d Mon Sep 17 00:00:00 2001
From: ashwinb <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 17:45:30 +0000
Subject: [PATCH 36/45] docs(tests): Add a bunch of documentation for our
 testing systems (#3139)

# What does this PR do?

Creates a structured testing documentation section with multiple detailed pages:

- Testing overview explaining the record-replay architecture
- Integration testing guide with practical usage examples
- Record-replay system technical documentation
- Guide for writing effective tests
- Troubleshooting guide for common testing issues

Hopefully this makes things a bit easier.
---
 docs/source/contributing/index.md             |  23 +-
 docs/source/contributing/testing.md           |   8 -
 .../contributing/testing/record-replay.md     | 234 ++++++++++++++++++
 tests/README.md                               |  91 ++++++-
 tests/integration/README.md                   | 189 +++++++++-----
 5 files changed, 456 insertions(+), 89 deletions(-)
 delete mode 100644 docs/source/contributing/testing.md
 create mode 100644 docs/source/contributing/testing/record-replay.md

diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md
index 7a3a1c2e2..296a49f24 100644
--- a/docs/source/contributing/index.md
+++ b/docs/source/contributing/index.md
@@ -4,11 +4,11 @@
 
 ## Adding a New Provider
 
-See the [Adding a New API Provider Page](new_api_provider.md) which describes how to add new API providers to the Stack.
+See:
+- [Adding a New API Provider Page](new_api_provider.md) which describes how to add new API providers to the Stack.
+- [Vector Database Page](new_vector_database.md) which describes how to add a new vector databases with Llama Stack.
+- [External Provider Page](../providers/external/index.md) which describes how to add external providers to the Stack.
 
-See the [Vector Database Page](new_vector_database.md) which describes how to add a new vector databases with Llama Stack.
-
-See the [External Provider Page](../providers/external/index.md) which describes how to add external providers to the Stack.
 ```{toctree}
 :maxdepth: 1
 :hidden:
@@ -19,11 +19,16 @@ new_vector_database
 
 ## Testing
 
-See the [Test Page](testing.md) which describes how to test your changes.
+
+```{include} ../../../tests/README.md
+```
+
+### Advanced Topics
+
+For developers who need deeper understanding of the testing system internals:
+
 ```{toctree}
 :maxdepth: 1
-:hidden:
-:caption: Testing
 
-testing
-```
\ No newline at end of file
+testing/record-replay
+```
diff --git a/docs/source/contributing/testing.md b/docs/source/contributing/testing.md
deleted file mode 100644
index 454ded266..000000000
--- a/docs/source/contributing/testing.md
+++ /dev/null
@@ -1,8 +0,0 @@
-```{include} ../../../tests/README.md
-```
-
-```{include} ../../../tests/unit/README.md
-```
-
-```{include} ../../../tests/integration/README.md
-```
diff --git a/docs/source/contributing/testing/record-replay.md b/docs/source/contributing/testing/record-replay.md
new file mode 100644
index 000000000..3049d333c
--- /dev/null
+++ b/docs/source/contributing/testing/record-replay.md
@@ -0,0 +1,234 @@
+# Record-Replay System
+
+Understanding how Llama Stack captures and replays API interactions for testing.
+
+## Overview
+
+The record-replay system solves a fundamental challenge in AI testing: how do you test against expensive, non-deterministic APIs without breaking the bank or dealing with flaky tests?
+
+The solution: intercept API calls, store real responses, and replay them later. This gives you real API behavior without the cost or variability.
+
+## How It Works
+
+### Request Hashing
+
+Every API request gets converted to a deterministic hash for lookup:
+
+```python
+def normalize_request(method: str, url: str, headers: dict, body: dict) -> str:
+    normalized = {
+        "method": method.upper(),
+        "endpoint": urlparse(url).path,  # Just the path, not full URL
+        "body": body,  # Request parameters
+    }
+    return hashlib.sha256(json.dumps(normalized, sort_keys=True).encode()).hexdigest()
+```
+
+**Key insight:** The hashing is intentionally precise. Different whitespace, float precision, or parameter order produces different hashes. This prevents subtle bugs from false cache hits.
+
+```python
+# These produce DIFFERENT hashes:
+{"content": "Hello world"}
+{"content": "Hello   world\n"}
+{"temperature": 0.7}
+{"temperature": 0.7000001}
+```
+
+### Client Interception
+
+The system patches OpenAI and Ollama client methods to intercept calls before they leave your application. This happens transparently - your test code doesn't change.
+
+### Storage Architecture
+
+Recordings use a two-tier storage system optimized for both speed and debuggability:
+
+```
+recordings/
+├── index.sqlite          # Fast lookup by request hash
+└── responses/
+    ├── abc123def456.json  # Individual response files
+    └── def789ghi012.json
+```
+
+**SQLite index** enables O(log n) hash lookups and metadata queries without loading response bodies.
+
+**JSON files** store complete request/response pairs in human-readable format for debugging.
+
+## Recording Modes
+
+### LIVE Mode
+
+Direct API calls with no recording or replay:
+
+```python
+with inference_recording(mode=InferenceMode.LIVE):
+    response = await client.chat.completions.create(...)
+```
+
+Use for initial development and debugging against real APIs.
+
+### RECORD Mode
+
+Captures API interactions while passing through real responses:
+
+```python
+with inference_recording(mode=InferenceMode.RECORD, storage_dir="./recordings"):
+    response = await client.chat.completions.create(...)
+    # Real API call made, response captured AND returned
+```
+
+The recording process:
+1. Request intercepted and hashed
+2. Real API call executed
+3. Response captured and serialized
+4. Recording stored to disk
+5. Original response returned to caller
+
+### REPLAY Mode
+
+Returns stored responses instead of making API calls:
+
+```python
+with inference_recording(mode=InferenceMode.REPLAY, storage_dir="./recordings"):
+    response = await client.chat.completions.create(...)
+    # No API call made, cached response returned instantly
+```
+
+The replay process:
+1. Request intercepted and hashed
+2. Hash looked up in SQLite index
+3. Response loaded from JSON file
+4. Response deserialized and returned
+5. Error if no recording found
+
+## Streaming Support
+
+Streaming APIs present a unique challenge: how do you capture an async generator?
+
+### The Problem
+
+```python
+# How do you record this?
+async for chunk in client.chat.completions.create(stream=True):
+    process(chunk)
+```
+
+### The Solution
+
+The system captures all chunks immediately before yielding any:
+
+```python
+async def handle_streaming_record(response):
+    # Capture complete stream first
+    chunks = []
+    async for chunk in response:
+        chunks.append(chunk)
+
+    # Store complete recording
+    storage.store_recording(
+        request_hash, request_data, {"body": chunks, "is_streaming": True}
+    )
+
+    # Return generator that replays captured chunks
+    async def replay_stream():
+        for chunk in chunks:
+            yield chunk
+
+    return replay_stream()
+```
+
+This ensures:
+- **Complete capture** - The entire stream is saved atomically
+- **Interface preservation** - The returned object behaves like the original API
+- **Deterministic replay** - Same chunks in the same order every time
+
+## Serialization
+
+API responses contain complex Pydantic objects that need careful serialization:
+
+```python
+def _serialize_response(response):
+    if hasattr(response, "model_dump"):
+        # Preserve type information for proper deserialization
+        return {
+            "__type__": f"{response.__class__.__module__}.{response.__class__.__qualname__}",
+            "__data__": response.model_dump(mode="json"),
+        }
+    return response
+```
+
+This preserves type safety - when replayed, you get the same Pydantic objects with all their validation and methods.
+
+## Environment Integration
+
+### Environment Variables
+
+Control recording behavior globally:
+
+```bash
+export LLAMA_STACK_TEST_INFERENCE_MODE=replay
+export LLAMA_STACK_TEST_RECORDING_DIR=/path/to/recordings
+pytest tests/integration/
+```
+
+### Pytest Integration
+
+The system integrates automatically based on environment variables, requiring no changes to test code.
+
+## Debugging Recordings
+
+### Inspecting Storage
+
+```bash
+# See what's recorded
+sqlite3 recordings/index.sqlite "SELECT endpoint, model, timestamp FROM recordings LIMIT 10;"
+
+# View specific response
+cat recordings/responses/abc123def456.json | jq '.response.body'
+
+# Find recordings by endpoint
+sqlite3 recordings/index.sqlite "SELECT * FROM recordings WHERE endpoint='/v1/chat/completions';"
+```
+
+### Common Issues
+
+**Hash mismatches:** Request parameters changed slightly between record and replay
+```bash
+# Compare request details
+cat recordings/responses/abc123.json | jq '.request'
+```
+
+**Serialization errors:** Response types changed between versions
+```bash
+# Re-record with updated types
+rm recordings/responses/failing_hash.json
+LLAMA_STACK_TEST_INFERENCE_MODE=record pytest test_failing.py
+```
+
+**Missing recordings:** New test or changed parameters
+```bash
+# Record the missing interaction
+LLAMA_STACK_TEST_INFERENCE_MODE=record pytest test_new.py
+```
+
+## Design Decisions
+
+### Why Not Mocks?
+
+Traditional mocking breaks down with AI APIs because:
+- Response structures are complex and evolve frequently
+- Streaming behavior is hard to mock correctly
+- Edge cases in real APIs get missed
+- Mocks become brittle maintenance burdens
+
+### Why Precise Hashing?
+
+Loose hashing (normalizing whitespace, rounding floats) seems convenient but hides bugs. If a test changes slightly, you want to know about it rather than accidentally getting the wrong cached response.
+
+### Why JSON + SQLite?
+
+- **JSON** - Human readable, diff-friendly, easy to inspect and modify
+- **SQLite** - Fast indexed lookups without loading response bodies
+- **Hybrid** - Best of both worlds for different use cases
+
+This system provides reliable, fast testing against real AI APIs while maintaining the ability to debug issues when they arise.
\ No newline at end of file
diff --git a/tests/README.md b/tests/README.md
index ed7064bfb..abbfc6d60 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,9 +1,86 @@
-# Llama Stack Tests
+There are two obvious types of tests:
 
-Llama Stack has multiple layers of testing done to ensure continuous functionality and prevent regressions to the codebase.
+| Type | Location | Purpose |
+|------|----------|---------|
+| **Unit** | [`tests/unit/`](unit/README.md) | Fast, isolated component testing |
+| **Integration** | [`tests/integration/`](integration/README.md) | End-to-end workflows with record-replay |
 
-| Testing Type | Details |
-|--------------|---------|
-| Unit | [unit/README.md](unit/README.md) |
-| Integration | [integration/README.md](integration/README.md) |
-| Verification | [verifications/README.md](verifications/README.md) |
+Both have their place. For unit tests, it is important to create minimal mocks and instead rely more on "fakes". Mocks are too brittle. In either case, tests must be very fast and reliable.
+
+### Record-replay for integration tests
+
+Testing AI applications end-to-end creates some challenges:
+- **API costs** accumulate quickly during development and CI
+- **Non-deterministic responses** make tests unreliable
+- **Multiple providers** require testing the same logic across different APIs
+
+Our solution: **Record real API responses once, replay them for fast, deterministic tests.** This is better than mocking because AI APIs have complex response structures and streaming behavior. Mocks can miss edge cases that real APIs exhibit. A single test can exercise underlying APIs in multiple complex ways making it really hard to mock.
+
+This gives you:
+- Cost control - No repeated API calls during development
+- Speed - Instant test execution with cached responses
+- Reliability - Consistent results regardless of external service state
+- Provider coverage - Same tests work across OpenAI, Anthropic, local models, etc.
+
+### Testing Quick Start
+
+You can run the unit tests with:
+```bash
+uv run --group unit pytest -sv tests/unit/
+```
+
+For running integration tests, you must provide a few things:
+
+- A stack config. This is a pointer to a stack. You have a few ways to point to a stack:
+  - **`server:<config>`** - automatically start a server with the given config (e.g., `server:starter`). This provides one-step testing by auto-starting the server if the port is available, or reusing an existing server if already running.
+  - **`server:<config>:<port>`** - same as above but with a custom port (e.g., `server:starter:8322`)
+  - a URL which points to a Llama Stack distribution server
+  - a distribution name (e.g., `starter`) or a path to a `run.yaml` file
+  - a comma-separated list of api=provider pairs, e.g. `inference=fireworks,safety=llama-guard,agents=meta-reference`. This is most useful for testing a single API surface.
+
+- Whether you are using replay or live mode for inference. This is specified with the LLAMA_STACK_TEST_INFERENCE_MODE environment variable. The default mode currently is "live" -- that is certainly surprising, but we will fix this soon.
+
+- Any API keys you need to use should be set in the environment, or can be passed in with the --env option.
+
+You can run the integration tests in replay mode with:
+```bash
+# Run all tests with existing recordings
+LLAMA_STACK_TEST_INFERENCE_MODE=replay \
+  LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings \
+  uv run --group test \
+  pytest -sv tests/integration/ --stack-config=starter
+```
+
+If you don't specify LLAMA_STACK_TEST_INFERENCE_MODE, by default it will be in "live" mode -- that is, it will make real API calls.
+
+```bash
+# Test against live APIs
+FIREWORKS_API_KEY=your_key pytest -sv tests/integration/inference --stack-config=starter
+```
+
+### Re-recording tests
+
+If you want to re-record tests, you can do so with:
+
+```bash
+LLAMA_STACK_TEST_INFERENCE_MODE=record \
+  LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings \
+  uv run --group test \
+  pytest -sv tests/integration/ --stack-config=starter -k "<appropriate test name>"
+```
+
+This will record new API responses and overwrite the existing recordings.
+
+
+```{warning}
+
+You must be careful when re-recording. CI workflows assume a specific setup for running the replay-mode tests. You must re-record the tests in the same way as the CI workflows. This means
+- you need Ollama running and serving some specific models.
+- you are using the `starter` distribution.
+```
+
+
+### Next Steps
+
+- [Integration Testing Guide](integration/README.md) - Detailed usage and configuration
+- [Unit Testing Guide](unit/README.md) - Fast component testing
diff --git a/tests/integration/README.md b/tests/integration/README.md
index 664116bea..427b905b4 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -1,6 +1,20 @@
-# Llama Stack Integration Tests
+# Integration Testing Guide
 
-We use `pytest` for parameterizing and running tests. You can see all options with:
+Integration tests verify complete workflows across different providers using Llama Stack's record-replay system.
+
+## Quick Start
+
+```bash
+# Run all integration tests with existing recordings
+LLAMA_STACK_TEST_INFERENCE_MODE=replay \
+  LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings \
+  uv run --group test \
+  pytest -sv tests/integration/ --stack-config=starter
+```
+
+## Configuration Options
+
+You can see all options with:
 ```bash
 cd tests/integration
 
@@ -10,11 +24,11 @@ pytest --help
 
 Here are the most important options:
 - `--stack-config`: specify the stack config to use. You have four ways to point to a stack:
-  - **`server:<config>`** - automatically start a server with the given config (e.g., `server:fireworks`). This provides one-step testing by auto-starting the server if the port is available, or reusing an existing server if already running.
-  - **`server:<config>:<port>`** - same as above but with a custom port (e.g., `server:together:8322`)
+  - **`server:<config>`** - automatically start a server with the given config (e.g., `server:starter`). This provides one-step testing by auto-starting the server if the port is available, or reusing an existing server if already running.
+  - **`server:<config>:<port>`** - same as above but with a custom port (e.g., `server:starter:8322`)
   - a URL which points to a Llama Stack distribution server
-  - a template (e.g., `starter`) or a path to a `run.yaml` file
-  - a comma-separated list of api=provider pairs, e.g. `inference=fireworks,safety=llama-guard,agents=meta-reference`. This is most useful for testing a single API surface.
+  - a distribution name (e.g., `starter`) or a path to a `run.yaml` file
+  - a comma-separated list of api=provider pairs, e.g. `inference=ollama,safety=llama-guard,agents=meta-reference`. This is most useful for testing a single API surface.
 - `--env`: set environment variables, e.g. --env KEY=value. this is a utility option to set environment variables required by various providers.
 
 Model parameters can be influenced by the following options:
@@ -32,85 +46,130 @@ if no model is specified.
 
 ### Testing against a Server
 
-Run all text inference tests by auto-starting a server with the `fireworks` config:
+Run all text inference tests by auto-starting a server with the `starter` config:
 
 ```bash
-pytest -s -v tests/integration/inference/test_text_inference.py \
-   --stack-config=server:fireworks \
-   --text-model=meta-llama/Llama-3.1-8B-Instruct
+OLLAMA_URL=http://localhost:11434 \
+  pytest -s -v tests/integration/inference/test_text_inference.py \
+   --stack-config=server:starter \
+   --text-model=ollama/llama3.2:3b-instruct-fp16 \
+   --embedding-model=sentence-transformers/all-MiniLM-L6-v2
 ```
 
 Run tests with auto-server startup on a custom port:
 
 ```bash
-pytest -s -v tests/integration/inference/ \
-   --stack-config=server:together:8322 \
-   --text-model=meta-llama/Llama-3.1-8B-Instruct
-```
-
-Run multiple test suites with auto-server (eliminates manual server management):
-
-```bash
-# Auto-start server and run all integration tests
-export FIREWORKS_API_KEY=<your_key>
-
-pytest -s -v tests/integration/inference/ tests/integration/safety/ tests/integration/agents/ \
-   --stack-config=server:fireworks \
-   --text-model=meta-llama/Llama-3.1-8B-Instruct
+OLLAMA_URL=http://localhost:11434 \
+  pytest -s -v tests/integration/inference/ \
+   --stack-config=server:starter:8322 \
+   --text-model=ollama/llama3.2:3b-instruct-fp16 \
+   --embedding-model=sentence-transformers/all-MiniLM-L6-v2
 ```
 
 ### Testing with Library Client
 
-Run all text inference tests with the `starter` distribution using the `together` provider:
+The library client constructs the Stack "in-process" instead of using a server. This is useful during the iterative development process since you don't need to constantly start and stop servers.
+
+
+You can do this by simply using `--stack-config=starter` instead of `--stack-config=server:starter`.
+
+
+### Using ad-hoc distributions
+
+Sometimes, you may want to make up a distribution on the fly. This is useful for testing a single provider or a single API or a small combination of providers. You can do so by specifying a comma-separated list of api=provider pairs to the `--stack-config` option, e.g. `inference=remote::ollama,safety=inline::llama-guard,agents=inline::meta-reference`.
 
 ```bash
-ENABLE_TOGETHER=together pytest -s -v tests/integration/inference/test_text_inference.py \
-   --stack-config=starter \
-   --text-model=meta-llama/Llama-3.1-8B-Instruct
-```
-
-Run all text inference tests with the `starter` distribution using the `together` provider and `meta-llama/Llama-3.1-8B-Instruct`:
-
-```bash
-ENABLE_TOGETHER=together pytest -s -v tests/integration/inference/test_text_inference.py \
-   --stack-config=starter \
-   --text-model=meta-llama/Llama-3.1-8B-Instruct
-```
-
-Running all inference tests for a number of models using the `together` provider:
-
-```bash
-TEXT_MODELS=meta-llama/Llama-3.1-8B-Instruct,meta-llama/Llama-3.1-70B-Instruct
-VISION_MODELS=meta-llama/Llama-3.2-11B-Vision-Instruct
-EMBEDDING_MODELS=all-MiniLM-L6-v2
-ENABLE_TOGETHER=together
-export TOGETHER_API_KEY=<together_api_key>
-
 pytest -s -v tests/integration/inference/ \
-   --stack-config=together \
+   --stack-config=inference=remote::ollama,safety=inline::llama-guard,agents=inline::meta-reference \
    --text-model=$TEXT_MODELS \
    --vision-model=$VISION_MODELS \
    --embedding-model=$EMBEDDING_MODELS
 ```
 
-Same thing but instead of using the distribution, use an adhoc stack with just one provider (`fireworks` for inference):
+Another example: Running Vector IO tests for embedding models:
 
 ```bash
-export FIREWORKS_API_KEY=<fireworks_api_key>
-
-pytest -s -v tests/integration/inference/ \
-   --stack-config=inference=fireworks \
-   --text-model=$TEXT_MODELS \
-   --vision-model=$VISION_MODELS \
-   --embedding-model=$EMBEDDING_MODELS
-```
-
-Running Vector IO tests for a number of embedding models:
-
-```bash
-EMBEDDING_MODELS=all-MiniLM-L6-v2
-
 pytest -s -v tests/integration/vector_io/ \
-   --stack-config=inference=sentence-transformers,vector_io=sqlite-vec \
-   --embedding-model=$EMBEDDING_MODELS
+   --stack-config=inference=inline::sentence-transformers,vector_io=inline::sqlite-vec \
+   --embedding-model=sentence-transformers/all-MiniLM-L6-v2
+```
+
+## Recording Modes
+
+The testing system supports three modes controlled by environment variables:
+
+### LIVE Mode (Default)
+Tests make real API calls:
+```bash
+LLAMA_STACK_TEST_INFERENCE_MODE=live pytest tests/integration/
+```
+
+### RECORD Mode
+Captures API interactions for later replay:
+```bash
+LLAMA_STACK_TEST_INFERENCE_MODE=record \
+LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings \
+pytest tests/integration/inference/test_new_feature.py
+```
+
+### REPLAY Mode
+Uses cached responses instead of making API calls:
+```bash
+LLAMA_STACK_TEST_INFERENCE_MODE=replay \
+LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings \
+pytest tests/integration/
+```
+
+Note that right now you must specify the recording directory. This is because different tests use different recording directories and we don't (yet) have a fool-proof way to map a test to a recording directory. We are working on this.
+
+## Managing Recordings
+
+### Viewing Recordings
+```bash
+# See what's recorded
+sqlite3 recordings/index.sqlite "SELECT endpoint, model, timestamp FROM recordings;"
+
+# Inspect specific response
+cat recordings/responses/abc123.json | jq '.'
+```
+
+### Re-recording Tests
+```bash
+# Re-record specific tests
+LLAMA_STACK_TEST_INFERENCE_MODE=record \
+LLAMA_STACK_TEST_RECORDING_DIR=tests/integration/recordings \
+pytest -s -v --stack-config=server:starter tests/integration/inference/test_modified.py
+```
+
+Note that when re-recording tests, you must use a Stack pointing to a server (i.e., `server:starter`). This subtlety exists because the set of tests run in server are a superset of the set of tests run in the library client.
+
+## Writing Tests
+
+### Basic Test Pattern
+```python
+def test_basic_completion(llama_stack_client, text_model_id):
+    response = llama_stack_client.inference.completion(
+        model_id=text_model_id,
+        content=CompletionMessage(role="user", content="Hello"),
+    )
+
+    # Test structure, not AI output quality
+    assert response.completion_message is not None
+    assert isinstance(response.completion_message.content, str)
+    assert len(response.completion_message.content) > 0
+```
+
+### Provider-Specific Tests
+```python
+def test_asymmetric_embeddings(llama_stack_client, embedding_model_id):
+    if embedding_model_id not in MODELS_SUPPORTING_TASK_TYPE:
+        pytest.skip(f"Model {embedding_model_id} doesn't support task types")
+
+    query_response = llama_stack_client.inference.embeddings(
+        model_id=embedding_model_id,
+        contents=["What is machine learning?"],
+        task_type="query",
+    )
+
+    assert query_response.embeddings is not None
 ```

From e743d3fdf65937aead265dc2c44b9c5a9ac7bb60 Mon Sep 17 00:00:00 2001
From: Aakanksha Duggal <aduggal@redhat.com>
Date: Fri, 15 Aug 2025 13:51:41 -0400
Subject: [PATCH 37/45] refactor(agents): migrate to OpenAI chat completions
 API (#3097)

Replace chat_completion calls with openai_chat_completion to eliminate
dependency on legacy inference APIs.

# What does this PR do?
<!-- Provide a short summary of what this PR does and why. Link to
relevant issues if applicable. -->

<!-- If resolving an issue, uncomment and update the line below -->
 Closes #3067

## Test Plan
<!-- Describe the tests you ran to verify your changes with result
summaries. *Provide clear instructions so the plan can be easily
re-executed.* -->
---
 .../agents/meta_reference/agent_instance.py   | 67 ++++++++++++++++---
 1 file changed, 58 insertions(+), 9 deletions(-)

diff --git a/llama_stack/providers/inline/agents/meta_reference/agent_instance.py b/llama_stack/providers/inline/agents/meta_reference/agent_instance.py
index 5f7c90879..e9f89f8d2 100644
--- a/llama_stack/providers/inline/agents/meta_reference/agent_instance.py
+++ b/llama_stack/providers/inline/agents/meta_reference/agent_instance.py
@@ -68,6 +68,11 @@ from llama_stack.models.llama.datatypes import (
     BuiltinTool,
     ToolCall,
 )
+from llama_stack.providers.utils.inference.openai_compat import (
+    convert_message_to_openai_dict,
+    convert_openai_chat_completion_stream,
+    convert_tooldef_to_openai_tool,
+)
 from llama_stack.providers.utils.kvstore import KVStore
 from llama_stack.providers.utils.telemetry import tracing
 
@@ -510,16 +515,60 @@ class ChatAgent(ShieldRunnerMixin):
             async with tracing.span("inference") as span:
                 if self.agent_config.name:
                     span.set_attribute("agent_name", self.agent_config.name)
-                async for chunk in await self.inference_api.chat_completion(
-                    self.agent_config.model,
-                    input_messages,
-                    tools=self.tool_defs,
-                    tool_prompt_format=self.agent_config.tool_config.tool_prompt_format,
-                    response_format=self.agent_config.response_format,
+                # Convert messages to OpenAI format
+                openai_messages = []
+                for message in input_messages:
+                    openai_message = await convert_message_to_openai_dict(message)
+                    openai_messages.append(openai_message)
+
+                # Convert tool definitions to OpenAI format
+                openai_tools = None
+                if self.tool_defs:
+                    openai_tools = []
+                    for tool_def in self.tool_defs:
+                        openai_tool = convert_tooldef_to_openai_tool(tool_def)
+                        openai_tools.append(openai_tool)
+
+                # Extract tool_choice from tool_config for OpenAI compatibility
+                # Note: tool_choice can only be provided when tools are also provided
+                tool_choice = None
+                if openai_tools and self.agent_config.tool_config and self.agent_config.tool_config.tool_choice:
+                    tool_choice = (
+                        self.agent_config.tool_config.tool_choice.value
+                        if hasattr(self.agent_config.tool_config.tool_choice, "value")
+                        else str(self.agent_config.tool_config.tool_choice)
+                    )
+
+                # Convert sampling params to OpenAI format (temperature, top_p, max_tokens)
+                temperature = None
+                top_p = None
+                max_tokens = None
+                if sampling_params:
+                    if hasattr(sampling_params.strategy, "temperature"):
+                        temperature = sampling_params.strategy.temperature
+                    if hasattr(sampling_params.strategy, "top_p"):
+                        top_p = sampling_params.strategy.top_p
+                    if sampling_params.max_tokens:
+                        max_tokens = sampling_params.max_tokens
+
+                # Use OpenAI chat completion
+                openai_stream = await self.inference_api.openai_chat_completion(
+                    model=self.agent_config.model,
+                    messages=openai_messages,
+                    tools=openai_tools if openai_tools else None,
+                    tool_choice=tool_choice,
+                    temperature=temperature,
+                    top_p=top_p,
+                    max_tokens=max_tokens,
                     stream=True,
-                    sampling_params=sampling_params,
-                    tool_config=self.agent_config.tool_config,
-                ):
+                )
+
+                # Convert OpenAI stream back to Llama Stack format
+                response_stream = convert_openai_chat_completion_stream(
+                    openai_stream, enable_incremental_tool_calls=True
+                )
+
+                async for chunk in response_stream:
                     event = chunk.event
                     if event.event_type == ChatCompletionResponseEventType.start:
                         continue

From a275282685df0a467fe267f9848b78b38f01360a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Aug 2025 10:54:09 -0700
Subject: [PATCH 38/45] chore(python-deps): bump pymilvus from 2.5.14 to 2.6.0
 (#3086)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [pymilvus](https://github.com/milvus-io/pymilvus) from 2.5.14 to
2.6.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/milvus-io/pymilvus/releases">pymilvus's
releases</a>.</em></p>
<blockquote>
<h2>PyMilvus v2.6.0 Release Notes</h2>
<h2>New Features</h2>
<ol>
<li>Add APIs in MilvusClient</li>
</ol>
<ul>
<li>enhance: add describe and alter database in MilvusClient by <a
href="https://github.com/smellthemoon"><code>@​smellthemoon</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2433">milvus-io/pymilvus#2433</a></li>
<li>enhance: support milvus-client iterator by <a
href="https://github.com/MrPresent-Han"><code>@​MrPresent-Han</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2461">milvus-io/pymilvus#2461</a></li>
<li>enhance: Enable resource group api in milvus client by <a
href="https://github.com/weiliu1031"><code>@​weiliu1031</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2513">milvus-io/pymilvus#2513</a></li>
<li>enhance: add release_collection, drop_index, create_partition,
drop_partition, load_partition and release_partition by <a
href="https://github.com/brcarry"><code>@​brcarry</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2525">milvus-io/pymilvus#2525</a></li>
<li>enhance: enable describe_replica api in milvus client by <a
href="https://github.com/weiliu1031"><code>@​weiliu1031</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2541">milvus-io/pymilvus#2541</a></li>
<li>enhance: support recalls for milvus_client by <a
href="https://github.com/chasingegg"><code>@​chasingegg</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2552">milvus-io/pymilvus#2552</a></li>
<li>enhance: add use_database by <a
href="https://github.com/czs007"><code>@​czs007</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2491">milvus-io/pymilvus#2491</a></li>
</ul>
<ol start="2">
<li>Add AsyncMilvusClient</li>
</ol>
<ul>
<li>[FEAT] Asyncio support by <a
href="https://github.com/brcarry"><code>@​brcarry</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2411">milvus-io/pymilvus#2411</a></li>
<li>Add async DDL funcs &amp; DDL examples by <a
href="https://github.com/Shawnzheng011019"><code>@​Shawnzheng011019</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2852">milvus-io/pymilvus#2852</a></li>
</ul>
<ol start="3">
<li>Other features</li>
</ol>
<ul>
<li>enhance: support Int8Vector by <a
href="https://github.com/cydrain"><code>@​cydrain</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2611">milvus-io/pymilvus#2611</a></li>
<li>feat: support recalls field in SearchResult by <a
href="https://github.com/chasingegg"><code>@​chasingegg</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2390">milvus-io/pymilvus#2390</a></li>
<li>enhance: Support Python3.13 and upgrade grpcio range by <a
href="https://github.com/XuanYang-cn"><code>@​XuanYang-cn</code></a> in
<a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2684">milvus-io/pymilvus#2684</a></li>
<li>enhance: support run analyzer return detail token by <a
href="https://github.com/aoiasd"><code>@​aoiasd</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2679">milvus-io/pymilvus#2679</a></li>
<li>enhance: Add force_drop parameter to drop_role method for role
deletion by <a href="https://github.com/SimFG"><code>@​SimFG</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2705">milvus-io/pymilvus#2705</a></li>
<li>enhance: add property func for AnalyzeToken by <a
href="https://github.com/aoiasd"><code>@​aoiasd</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2704">milvus-io/pymilvus#2704</a></li>
<li>enhance: grant/revoke v2 optional db and collection params by <a
href="https://github.com/shaoting-huang"><code>@​shaoting-huang</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2386">milvus-io/pymilvus#2386</a></li>
<li>extend unlimted offset for query iterator(<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2418">#2418</a>)
by <a
href="https://github.com/MrPresent-Han"><code>@​MrPresent-Han</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2419">milvus-io/pymilvus#2419</a></li>
<li>enhance: alterindex &amp; altercollection supports altering
properties by <a
href="https://github.com/JsDove"><code>@​JsDove</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2406">milvus-io/pymilvus#2406</a></li>
<li>enhance: alterdatabase support delete property by <a
href="https://github.com/JsDove"><code>@​JsDove</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2435">milvus-io/pymilvus#2435</a></li>
<li>enhance: support hints param by <a
href="https://github.com/chasingegg"><code>@​chasingegg</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2408">milvus-io/pymilvus#2408</a></li>
<li>enhance: create database support properties by <a
href="https://github.com/JsDove"><code>@​JsDove</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2448">milvus-io/pymilvus#2448</a></li>
<li>enhance: Add <code>db_name</code> parameter at
<code>bulk_import</code> by <a
href="https://github.com/counter2015"><code>@​counter2015</code></a> in
<a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2446">milvus-io/pymilvus#2446</a></li>
<li>enhance: add search iterator v2 by <a
href="https://github.com/PwzXxm"><code>@​PwzXxm</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2395">milvus-io/pymilvus#2395</a></li>
<li>enhance: simplify the structure of search_params by <a
href="https://github.com/smellthemoon"><code>@​smellthemoon</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2507">milvus-io/pymilvus#2507</a></li>
<li>enhance: Remove long deprecated Milvus class by <a
href="https://github.com/XuanYang-cn"><code>@​XuanYang-cn</code></a> in
<a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2544">milvus-io/pymilvus#2544</a></li>
<li>enhance: Use new model pkg by <a
href="https://github.com/junjiejiangjjj"><code>@​junjiejiangjjj</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2595">milvus-io/pymilvus#2595</a></li>
<li>enhance: Add schema update time verification to insert and upsert to
use cache by <a
href="https://github.com/JsDove"><code>@​JsDove</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2551">milvus-io/pymilvus#2551</a></li>
<li>enhance: describecollection output add created_timestamp by <a
href="https://github.com/JsDove"><code>@​JsDove</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2618">milvus-io/pymilvus#2618</a></li>
<li>feat: add external filter func for search iterator v2 by <a
href="https://github.com/PwzXxm"><code>@​PwzXxm</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2639">milvus-io/pymilvus#2639</a></li>
<li>enhance: support run analyzer by <a
href="https://github.com/aoiasd"><code>@​aoiasd</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2622">milvus-io/pymilvus#2622</a></li>
<li>weighted reranker to allow skip score normalization by <a
href="https://github.com/zhengbuqian"><code>@​zhengbuqian</code></a> in
<a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2708">milvus-io/pymilvus#2708</a></li>
<li>enhance: Support AddCollectionField API by <a
href="https://github.com/congqixia"><code>@​congqixia</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2722">milvus-io/pymilvus#2722</a></li>
<li>Add 1-Way and 2-Way TLS Support to Bulk Import Functions by <a
href="https://github.com/abd-770"><code>@​abd-770</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2672">milvus-io/pymilvus#2672</a></li>
<li>enhance: Use SearchResult in MilvusClient by <a
href="https://github.com/XuanYang-cn"><code>@​XuanYang-cn</code></a> in
<a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2735">milvus-io/pymilvus#2735</a></li>
<li>Support rerank by <a
href="https://github.com/junjiejiangjjj"><code>@​junjiejiangjjj</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2729">milvus-io/pymilvus#2729</a></li>
<li>feat: suppoprt multi analyzer params by <a
href="https://github.com/aoiasd"><code>@​aoiasd</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2747">milvus-io/pymilvus#2747</a></li>
<li>Add funciton checker by <a
href="https://github.com/junjiejiangjjj"><code>@​junjiejiangjjj</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2760">milvus-io/pymilvus#2760</a></li>
<li>enhance: Support run analyzer by collection and field by <a
href="https://github.com/aoiasd"><code>@​aoiasd</code></a> in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2822">milvus-io/pymilvus#2822</a></li>
<li>feat: support load collection/partition with priority(<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2835">#2835</a>)
by <a
href="https://github.com/MrPresent-Han"><code>@​MrPresent-Han</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2836">milvus-io/pymilvus#2836</a></li>
<li>enhance: optimize perf for large topk(<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2848">#2848</a>)
by <a
href="https://github.com/MrPresent-Han"><code>@​MrPresent-Han</code></a>
in <a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2849">milvus-io/pymilvus#2849</a></li>
<li>enhance: Add usage guide to manage MilvusClient by <a
href="https://github.com/XuanYang-cn"><code>@​XuanYang-cn</code></a> in
<a
href="https://redirect.github.com/milvus-io/pymilvus/pull/2907">milvus-io/pymilvus#2907</a></li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/1e56ce7d3159883108fb581d820572c1ab9a4c11"><code>1e56ce7</code></a>
enhance: Update milvus-proto and readme (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2921">#2921</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/75052b1b7cada14585eed4e7d7aa68ac2f541306"><code>75052b1</code></a>
enhance: Add usage guide to manage MilvusClient (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2907">#2907</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/9f44053086002d782cf7f81576a7a0f2bcc9f5ad"><code>9f44053</code></a>
add example code for language identifier and multi analyzer (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2919">#2919</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/058836de26f84c4c9e82490e3ef76a6a122b8dff"><code>058836d</code></a>
fix: Return new pk value for upsert when autoid=true (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2914">#2914</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/bbc6777565694dc2d6479177c5d2136256d332a9"><code>bbc6777</code></a>
[cherry-pick] Compatible with the default behavior of free on the cloud
(<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2913">#2913</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/45080c39c59b5dbc8a5766d8afb9c08bba8d3896"><code>45080c3</code></a>
fix: Aviod coping functions when init CollectionSchema (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2902">#2902</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/52b8461c5bee1466c4678f3bc029a9cda4422bd0"><code>52b8461</code></a>
[cherry-pick] bulk_import add stageName/dataPaths parameter (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2905">#2905</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/a8c3120622290d429dc3b12ece643248dc2f5d76"><code>a8c3120</code></a>
[cherry-pick] support stage (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2895">#2895</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/3653effa88a8af263690ca88b53dc529da3082fb"><code>3653eff</code></a>
fix: Tidy alias configs when connect fails (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2900">#2900</a>)</li>
<li><a
href="https://github.com/milvus-io/pymilvus/commit/728791a7de7d6eb0a4201ba755ea0675bb6976a0"><code>728791a</code></a>
enhance: Store alias before wait for ready (<a
href="https://redirect.github.com/milvus-io/pymilvus/issues/2894">#2894</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/milvus-io/pymilvus/compare/v2.5.14...v2.6.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pymilvus&package-manager=uv&previous-version=2.5.14&new-version=2.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 uv.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uv.lock b/uv.lock
index 4c56816ef..d5a2602a4 100644
--- a/uv.lock
+++ b/uv.lock
@@ -3383,7 +3383,7 @@ wheels = [
 
 [[package]]
 name = "pymilvus"
-version = "2.5.14"
+version = "2.6.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "grpcio" },
@@ -3394,9 +3394,9 @@ dependencies = [
     { name = "setuptools" },
     { name = "ujson" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/bb/f5/ab9309bd59d141d7977512b870eb5286ec80ced450ecdc5580b06f5fdf1a/pymilvus-2.5.14.tar.gz", hash = "sha256:ba831aa79d29feb3a5ff846c07a59015d0f995949d0dfd2f420554cda0261b98", size = 1270850, upload-time = "2025-07-21T16:19:07.74Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/86/21/5c25a975299415a5a8f26d4759ddf7852aefdf3595f002b5203c4aaf5c8e/pymilvus-2.6.0.tar.gz", hash = "sha256:2b2ca487e098abc34231755e33af2f5294e9f6a64d92d03551532defbac0a3fb", size = 1292994, upload-time = "2025-08-06T09:09:01.705Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/58/39/e6574fa640583e33ab6e709d61bbad315130ca42dcbf449aa025c3789a63/pymilvus-2.5.14-py3-none-any.whl", hash = "sha256:0e3cb687fd0807770cafb59566d217998b2166edcfa11956dd6e3fbbe2136a0f", size = 236412, upload-time = "2025-07-21T16:19:05.556Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/a2/dfc2a2225aeb90a7dff9443f2d26fe9d04f6f7bcefe537945b5d5220fddd/pymilvus-2.6.0-py3-none-any.whl", hash = "sha256:d743fdd928c9007184d24a52b4f5dfdd18d405a37b4dba66b5ea4bf196fac526", size = 248299, upload-time = "2025-08-06T09:08:58.272Z" },
 ]
 
 [[package]]

From 2114214fe36afeeda1c16ee076456ffb62ae66b4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Aug 2025 10:55:43 -0700
Subject: [PATCH 39/45] chore(python-deps): bump huggingface-hub from 0.34.3 to
 0.34.4 (#3084)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [huggingface-hub](https://github.com/huggingface/huggingface_hub)
from 0.34.3 to 0.34.4.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/huggingface/huggingface_hub/releases">huggingface-hub's
releases</a>.</em></p>
<blockquote>
<h2>[v0.34.4] Support Image to Video inference + QoL in jobs API, auth
and utilities</h2>
<p>Biggest update is the support of Image-To-Video task with inference
provider Fal AI</p>
<ul>
<li>[Inference] Support image to video task <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3289">#3289</a>
by <a
href="https://github.com/hanouticelina"><code>@​hanouticelina</code></a></li>
</ul>
<pre lang="py"><code>&gt;&gt;&gt; from huggingface_hub import
InferenceClient
&gt;&gt;&gt; client = InferenceClient()
&gt;&gt;&gt; video = client.image_to_video(&quot;cat.jpg&quot;,
model=&quot;Wan-AI/Wan2.2-I2V-A14B&quot;, prompt=&quot;turn the cat into
a tiger&quot;)
&gt;&gt;&gt; with open(&quot;tiger.mp4&quot;, &quot;wb&quot;) as f:
 ...     f.write(video)
</code></pre>
<p>And some quality of life improvements:</p>
<ul>
<li>Add type to job owner <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3291">#3291</a>
by <a href="https://github.com/drbh"><code>@​drbh</code></a></li>
<li>Include HF_HUB_DISABLE_XET in the environment dump <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3290">#3290</a>
by <a
href="https://github.com/hanouticelina"><code>@​hanouticelina</code></a></li>
<li>Whoami: custom message only on unauthorized <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3288">#3288</a>
by <a href="https://github.com/Wauplin"><code>@​Wauplin</code></a></li>
<li>Add validation warnings for repository limits in upload_large_folder
<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3280">#3280</a>
by <a
href="https://github.com/davanstrien"><code>@​davanstrien</code></a></li>
<li>Add timeout info to Jobs guide docs <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3281">#3281</a>
by <a
href="https://github.com/davanstrien"><code>@​davanstrien</code></a></li>
<li>[Jobs] Use current or stored token in a Job secrets <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3272">#3272</a>
by <a href="https://github.com/lhoestq"><code>@​lhoestq</code></a></li>
<li>Fix bash history expansion in hf jobs example <a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3277">#3277</a>
by <a
href="https://github.com/nyuuzyou"><code>@​nyuuzyou</code></a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/huggingface/huggingface_hub/compare/v0.34.3...v0.34.4">https://github.com/huggingface/huggingface_hub/compare/v0.34.3...v0.34.4</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/84a92a92c24b7755963ab4195bf2c8b4129c911f"><code>84a92a9</code></a>
Release: v0.34.4</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/6196ac2cbc25ad7a4bce81bc1e6f6bd9a374d458"><code>6196ac2</code></a>
Add type to job owner (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3291">#3291</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/4f6975f697d43087fea5646854cf5d121f6ee211"><code>4f6975f</code></a>
Include <code>HF_HUB_DISABLE_XET</code> in the environment dump (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3290">#3290</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/3720a5096f18873eae36a063e162261598e680de"><code>3720a50</code></a>
[Inference] Support image to video task (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3289">#3289</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/bb5e4c7a2c63d68bdbb7682085ab9453f9e5287a"><code>bb5e4c7</code></a>
Whoami: custom message only on unauthorized (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3288">#3288</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/a725256f318bdd69e8e224ee535c50f68dd37bc1"><code>a725256</code></a>
Add validation warnings for repository limits in upload_large_folder (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3280">#3280</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/a181b0f088f0ee4721eefb2ed0de7191eb4ee6a2"><code>a181b0f</code></a>
Add timeout info to Jobs guide docs (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3281">#3281</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/4d38925c8dab2881469d64eddacef73eda9e8a32"><code>4d38925</code></a>
[Jobs] Use current or stored token in a Job secrets (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3272">#3272</a>)</li>
<li><a
href="https://github.com/huggingface/huggingface_hub/commit/1580ce18c7419576a033622a461f35bdf0be021c"><code>1580ce1</code></a>
Fix bash history expansion in hf jobs example (<a
href="https://redirect.github.com/huggingface/huggingface_hub/issues/3277">#3277</a>)</li>
<li>See full diff in <a
href="https://github.com/huggingface/huggingface_hub/compare/v0.34.3...v0.34.4">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=huggingface-hub&package-manager=uv&previous-version=0.34.3&new-version=0.34.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Ashwin Bharambe <ashwin.bharambe@gmail.com>
---
 uv.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uv.lock b/uv.lock
index d5a2602a4..a09406770 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1360,7 +1360,7 @@ wheels = [
 
 [[package]]
 name = "huggingface-hub"
-version = "0.34.3"
+version = "0.34.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "filelock" },
@@ -1372,9 +1372,9 @@ dependencies = [
     { name = "tqdm" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/91/b4/e6b465eca5386b52cf23cb6df8644ad318a6b0e12b4b96a7e0be09cbfbcc/huggingface_hub-0.34.3.tar.gz", hash = "sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853", size = 456800, upload-time = "2025-07-29T08:38:53.885Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/45/c9/bdbe19339f76d12985bc03572f330a01a93c04dffecaaea3061bdd7fb892/huggingface_hub-0.34.4.tar.gz", hash = "sha256:a4228daa6fb001be3f4f4bdaf9a0db00e1739235702848df00885c9b5742c85c", size = 459768, upload-time = "2025-08-08T09:14:52.365Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/59/a8/4677014e771ed1591a87b63a2392ce6923baf807193deef302dcfde17542/huggingface_hub-0.34.3-py3-none-any.whl", hash = "sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492", size = 558847, upload-time = "2025-07-29T08:38:51.904Z" },
+    { url = "https://files.pythonhosted.org/packages/39/7b/bb06b061991107cd8783f300adff3e7b7f284e330fd82f507f2a1417b11d/huggingface_hub-0.34.4-py3-none-any.whl", hash = "sha256:9b365d781739c93ff90c359844221beef048403f1bc1f1c123c191257c3c890a", size = 561452, upload-time = "2025-08-08T09:14:50.159Z" },
 ]
 
 [[package]]

From 2c06b24c7724aa5331e90bdb6351bda07ee4f0a7 Mon Sep 17 00:00:00 2001
From: ehhuang <ehhuang@users.noreply.github.com>
Date: Fri, 15 Aug 2025 11:24:29 -0700
Subject: [PATCH 40/45] test: benchmark scripts (#3160)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

# What does this PR do?
1. Add our own benchmark script instead of locust (doesn't support
measuring streaming latency well)
2. Simplify k8s deployment
3. Add a simple profile script for locally running server

## Test Plan
❮ ./run-benchmark.sh --target stack --duration 180 --concurrent 10

============================================================
BENCHMARK RESULTS
============================================================
Total time: 180.00s
Concurrent users: 10
Total requests: 1636
Successful requests: 1636
Failed requests: 0
Success rate: 100.0%
Requests per second: 9.09

Response Time Statistics:
  Mean: 1.095s
  Median: 1.721s
  Min: 0.136s
  Max: 3.218s
  Std Dev: 0.762s

Percentiles:
  P50: 1.721s
  P90: 1.751s
  P95: 1.756s
  P99: 1.796s

Time to First Token (TTFT) Statistics:
  Mean: 0.037s
  Median: 0.037s
  Min: 0.023s
  Max: 0.211s
  Std Dev: 0.011s

TTFT Percentiles:
  P50: 0.037s
  P90: 0.040s
  P95: 0.044s
  P99: 0.055s

Streaming Statistics:
  Mean chunks per response: 64.0
  Total chunks received: 104775
---
 docs/source/contributing/index.md             |   5 +
 .../distributions/k8s-benchmark/README.md     | 156 ++++++++++
 .../distributions/k8s-benchmark/apply.sh      |  23 +-
 .../distributions/k8s-benchmark/benchmark.py  | 268 ++++++++++++++++++
 .../k8s-benchmark/locust-k8s.yaml             | 131 ---------
 .../distributions/k8s-benchmark/locustfile.py |  78 -----
 .../k8s-benchmark/openai-mock-deployment.yaml |  52 ----
 .../k8s-benchmark/openai-mock-server.py       |   6 +-
 .../k8s-benchmark/profile_running_server.sh   |  52 ++++
 .../k8s-benchmark/run-benchmark.sh            | 148 ++++++++++
 .../k8s-benchmark/stack-configmap.yaml        |  10 -
 .../k8s-benchmark/stack-k8s.yaml.template     |   4 -
 .../k8s-benchmark/stack_run_config.yaml       |  28 --
 13 files changed, 633 insertions(+), 328 deletions(-)
 create mode 100644 docs/source/distributions/k8s-benchmark/README.md
 create mode 100644 docs/source/distributions/k8s-benchmark/benchmark.py
 delete mode 100644 docs/source/distributions/k8s-benchmark/locust-k8s.yaml
 delete mode 100644 docs/source/distributions/k8s-benchmark/locustfile.py
 delete mode 100644 docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml
 create mode 100755 docs/source/distributions/k8s-benchmark/profile_running_server.sh
 create mode 100755 docs/source/distributions/k8s-benchmark/run-benchmark.sh

diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md
index 296a49f24..24bf3f66c 100644
--- a/docs/source/contributing/index.md
+++ b/docs/source/contributing/index.md
@@ -23,6 +23,11 @@ new_vector_database
 ```{include} ../../../tests/README.md
 ```
 
+## Benchmarking
+
+```{include} ../../../docs/source/distributions/k8s-benchmark/README.md
+```
+
 ### Advanced Topics
 
 For developers who need deeper understanding of the testing system internals:
diff --git a/docs/source/distributions/k8s-benchmark/README.md b/docs/source/distributions/k8s-benchmark/README.md
new file mode 100644
index 000000000..42da4d466
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/README.md
@@ -0,0 +1,156 @@
+# Llama Stack Benchmark Suite on Kubernetes
+
+## Motivation
+
+Performance benchmarking is critical for understanding the overhead and characteristics of the Llama Stack abstraction layer compared to direct inference engines like vLLM.
+
+### Why This Benchmark Suite Exists
+
+**Performance Validation**: The Llama Stack provides a unified API layer across multiple inference providers, but this abstraction introduces potential overhead. This benchmark suite quantifies the performance impact by comparing:
+- Llama Stack inference (with vLLM backend)
+- Direct vLLM inference calls
+- Both under identical Kubernetes deployment conditions
+
+**Production Readiness Assessment**: Real-world deployments require understanding performance characteristics under load. This suite simulates concurrent user scenarios with configurable parameters (duration, concurrency, request patterns) to validate production readiness.
+
+**Regression Detection (TODO)**: As the Llama Stack evolves, this benchmark provides automated regression detection for performance changes. CI/CD pipelines can leverage these benchmarks to catch performance degradations before production deployments.
+
+**Resource Planning**: By measuring throughput, latency percentiles, and resource utilization patterns, teams can make informed decisions about:
+- Kubernetes resource allocation (CPU, memory, GPU)
+- Auto-scaling configurations
+- Cost optimization strategies
+
+### Key Metrics Captured
+
+The benchmark suite measures critical performance indicators:
+- **Throughput**: Requests per second under sustained load
+- **Latency Distribution**: P50, P95, P99 response times
+- **Time to First Token (TTFT)**: Critical for streaming applications
+- **Error Rates**: Request failures and timeout analysis
+
+This data enables data-driven architectural decisions and performance optimization efforts.
+
+## Setup
+
+**1. Deploy base k8s infrastructure:**
+```bash
+cd ../k8s
+./apply.sh
+```
+
+**2. Deploy benchmark components:**
+```bash
+cd ../k8s-benchmark
+./apply.sh
+```
+
+**3. Verify deployment:**
+```bash
+kubectl get pods
+# Should see: llama-stack-benchmark-server, vllm-server, etc.
+```
+
+## Quick Start
+
+### Basic Benchmarks
+
+**Benchmark Llama Stack (default):**
+```bash
+cd docs/source/distributions/k8s-benchmark/
+./run-benchmark.sh
+```
+
+**Benchmark vLLM direct:**
+```bash
+./run-benchmark.sh --target vllm
+```
+
+### Custom Configuration
+
+**Extended benchmark with high concurrency:**
+```bash
+./run-benchmark.sh --target vllm --duration 120 --concurrent 20
+```
+
+**Short test run:**
+```bash
+./run-benchmark.sh --target stack --duration 30 --concurrent 5
+```
+
+## Command Reference
+
+### run-benchmark.sh Options
+
+```bash
+./run-benchmark.sh [options]
+
+Options:
+  -t, --target <stack|vllm>     Target to benchmark (default: stack)
+  -d, --duration <seconds>      Duration in seconds (default: 60)
+  -c, --concurrent <users>      Number of concurrent users (default: 10)
+  -h, --help                    Show help message
+
+Examples:
+  ./run-benchmark.sh --target vllm              # Benchmark vLLM direct
+  ./run-benchmark.sh --target stack             # Benchmark Llama Stack
+  ./run-benchmark.sh -t vllm -d 120 -c 20       # vLLM with 120s, 20 users
+```
+
+## Local Testing
+
+### Running Benchmark Locally
+
+For local development without Kubernetes:
+
+**1. Start OpenAI mock server:**
+```bash
+uv run python openai-mock-server.py --port 8080
+```
+
+**2. Run benchmark against mock server:**
+```bash
+uv run python benchmark.py \
+  --base-url http://localhost:8080/v1 \
+  --model mock-inference \
+  --duration 30 \
+  --concurrent 5
+```
+
+**3. Test against local vLLM server:**
+```bash
+# If you have vLLM running locally on port 8000
+uv run python benchmark.py \
+  --base-url http://localhost:8000/v1 \
+  --model meta-llama/Llama-3.2-3B-Instruct \
+  --duration 30 \
+  --concurrent 5
+```
+
+**4. Profile the running server:**
+```bash
+./profile_running_server.sh
+```
+
+
+
+### OpenAI Mock Server
+
+The `openai-mock-server.py` provides:
+- **OpenAI-compatible API** for testing without real models
+- **Configurable streaming delay** via `STREAM_DELAY_SECONDS` env var
+- **Consistent responses** for reproducible benchmarks
+- **Lightweight testing** without GPU requirements
+
+**Mock server usage:**
+```bash
+uv run python openai-mock-server.py --port 8080
+```
+
+The mock server is also deployed in k8s as `openai-mock-service:8080` and can be used by changing the Llama Stack configuration to use the `mock-vllm-inference` provider.
+
+## Files in this Directory
+
+- `benchmark.py` - Core benchmark script with async streaming support
+- `run-benchmark.sh` - Main script with target selection and configuration
+- `openai-mock-server.py` - Mock OpenAI API server for local testing
+- `README.md` - This documentation file
diff --git a/docs/source/distributions/k8s-benchmark/apply.sh b/docs/source/distributions/k8s-benchmark/apply.sh
index 119a1c849..4f2270da8 100755
--- a/docs/source/distributions/k8s-benchmark/apply.sh
+++ b/docs/source/distributions/k8s-benchmark/apply.sh
@@ -8,7 +8,6 @@
 
 # Deploys the benchmark-specific components on top of the base k8s deployment (../k8s/apply.sh).
 
-export MOCK_INFERENCE_PORT=8080
 export STREAM_DELAY_SECONDS=0.005
 
 export POSTGRES_USER=llamastack
@@ -20,14 +19,7 @@ export SAFETY_MODEL=meta-llama/Llama-Guard-3-1B
 
 export MOCK_INFERENCE_MODEL=mock-inference
 
-# Use llama-stack-benchmark-service as the benchmark server
-export LOCUST_HOST=http://llama-stack-benchmark-service:8323
-export LOCUST_BASE_PATH=/v1/openai/v1
-
-# Use vllm-service as the benchmark server
-# export LOCUST_HOST=http://vllm-server:8000
-# export LOCUST_BASE_PATH=/v1
-
+export MOCK_INFERENCE_URL=openai-mock-service:8080
 
 export BENCHMARK_INFERENCE_MODEL=$INFERENCE_MODEL
 
@@ -35,13 +27,6 @@ set -euo pipefail
 set -x
 
 # Deploy benchmark-specific components
-# Deploy OpenAI mock server
-kubectl create configmap openai-mock --from-file=openai-mock-server.py \
-  --dry-run=client -o yaml | kubectl apply --validate=false -f -
-
-envsubst < openai-mock-deployment.yaml | kubectl apply --validate=false -f -
-
-# Create configmap with our custom stack config
 kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \
   --dry-run=client -o yaml > stack-configmap.yaml
 
@@ -49,9 +34,3 @@ kubectl apply --validate=false -f stack-configmap.yaml
 
 # Deploy our custom llama stack server (overriding the base one)
 envsubst < stack-k8s.yaml.template | kubectl apply --validate=false -f -
-
-# Deploy Locust load testing
-kubectl create configmap locust-script --from-file=locustfile.py \
-  --dry-run=client -o yaml | kubectl apply --validate=false -f -
-
-envsubst < locust-k8s.yaml | kubectl apply --validate=false -f -
diff --git a/docs/source/distributions/k8s-benchmark/benchmark.py b/docs/source/distributions/k8s-benchmark/benchmark.py
new file mode 100644
index 000000000..0e7368431
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/benchmark.py
@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Simple benchmark script for Llama Stack with OpenAI API compatibility.
+"""
+
+import argparse
+import asyncio
+import os
+import random
+import statistics
+import time
+from typing import Tuple
+import aiohttp
+
+
+class BenchmarkStats:
+    def __init__(self):
+        self.response_times = []
+        self.ttft_times = []
+        self.chunks_received = []
+        self.errors = []
+        self.success_count = 0
+        self.total_requests = 0
+        self.concurrent_users = 0
+        self.start_time = None
+        self.end_time = None
+        self._lock = asyncio.Lock()
+
+    async def add_result(self, response_time: float, chunks: int, ttft: float = None, error: str = None):
+        async with self._lock:
+            self.total_requests += 1
+            if error:
+                self.errors.append(error)
+            else:
+                self.success_count += 1
+                self.response_times.append(response_time)
+                self.chunks_received.append(chunks)
+                if ttft is not None:
+                    self.ttft_times.append(ttft)
+
+    def print_summary(self):
+        if not self.response_times:
+            print("No successful requests to report")
+            if self.errors:
+                print(f"Total errors: {len(self.errors)}")
+                print("First 5 errors:")
+                for error in self.errors[:5]:
+                    print(f"  {error}")
+            return
+
+        total_time = self.end_time - self.start_time
+        success_rate = (self.success_count / self.total_requests) * 100
+        
+        print(f"\n{'='*60}")
+        print(f"BENCHMARK RESULTS")
+        print(f"{'='*60}")
+        print(f"Total time: {total_time:.2f}s")
+        print(f"Concurrent users: {self.concurrent_users}")
+        print(f"Total requests: {self.total_requests}")
+        print(f"Successful requests: {self.success_count}")
+        print(f"Failed requests: {len(self.errors)}")
+        print(f"Success rate: {success_rate:.1f}%")
+        print(f"Requests per second: {self.success_count / total_time:.2f}")
+        
+        print(f"\nResponse Time Statistics:")
+        print(f"  Mean: {statistics.mean(self.response_times):.3f}s")
+        print(f"  Median: {statistics.median(self.response_times):.3f}s")
+        print(f"  Min: {min(self.response_times):.3f}s")
+        print(f"  Max: {max(self.response_times):.3f}s")
+        
+        if len(self.response_times) > 1:
+            print(f"  Std Dev: {statistics.stdev(self.response_times):.3f}s")
+            
+        percentiles = [50, 90, 95, 99]
+        sorted_times = sorted(self.response_times)
+        print(f"\nPercentiles:")
+        for p in percentiles:
+            idx = int(len(sorted_times) * p / 100) - 1
+            idx = max(0, min(idx, len(sorted_times) - 1))
+            print(f"  P{p}: {sorted_times[idx]:.3f}s")
+            
+        if self.ttft_times:
+            print(f"\nTime to First Token (TTFT) Statistics:")
+            print(f"  Mean: {statistics.mean(self.ttft_times):.3f}s")
+            print(f"  Median: {statistics.median(self.ttft_times):.3f}s")
+            print(f"  Min: {min(self.ttft_times):.3f}s")
+            print(f"  Max: {max(self.ttft_times):.3f}s")
+            
+            if len(self.ttft_times) > 1:
+                print(f"  Std Dev: {statistics.stdev(self.ttft_times):.3f}s")
+                
+            sorted_ttft = sorted(self.ttft_times)
+            print(f"\nTTFT Percentiles:")
+            for p in percentiles:
+                idx = int(len(sorted_ttft) * p / 100) - 1
+                idx = max(0, min(idx, len(sorted_ttft) - 1))
+                print(f"  P{p}: {sorted_ttft[idx]:.3f}s")
+            
+        if self.chunks_received:
+            print(f"\nStreaming Statistics:")
+            print(f"  Mean chunks per response: {statistics.mean(self.chunks_received):.1f}")
+            print(f"  Total chunks received: {sum(self.chunks_received)}")
+        
+        if self.errors:
+            print(f"\nErrors (showing first 5):")
+            for error in self.errors[:5]:
+                print(f"  {error}")
+
+
+class LlamaStackBenchmark:
+    def __init__(self, base_url: str, model_id: str):
+        self.base_url = base_url.rstrip('/')
+        self.model_id = model_id
+        self.headers = {"Content-Type": "application/json"}
+        self.test_messages = [
+            [{"role": "user", "content": "Hi"}],
+            [{"role": "user", "content": "What is the capital of France?"}],
+            [{"role": "user", "content": "Explain quantum physics in simple terms."}],
+            [{"role": "user", "content": "Write a short story about a robot learning to paint."}],
+            [
+                {"role": "user", "content": "What is machine learning?"},
+                {"role": "assistant", "content": "Machine learning is a subset of AI..."},
+                {"role": "user", "content": "Can you give me a practical example?"}
+            ]
+        ]
+
+
+    async def make_async_streaming_request(self) -> Tuple[float, int, float | None, str | None]:
+        """Make a single async streaming chat completion request."""
+        messages = random.choice(self.test_messages)
+        payload = {
+            "model": self.model_id,
+            "messages": messages,
+            "stream": True,
+            "max_tokens": 100
+        }
+        
+        start_time = time.time()
+        chunks_received = 0
+        ttft = None
+        error = None
+        
+        session = aiohttp.ClientSession()
+        
+        try:
+            async with session.post(
+                f"{self.base_url}/chat/completions",
+                headers=self.headers,
+                json=payload,
+                timeout=aiohttp.ClientTimeout(total=30)
+            ) as response:
+                if response.status == 200:
+                    async for line in response.content:
+                        if line:
+                            line_str = line.decode('utf-8').strip()
+                            if line_str.startswith('data: '):
+                                chunks_received += 1
+                                if ttft is None:
+                                    ttft = time.time() - start_time
+                                if line_str == 'data: [DONE]':
+                                    break
+                    
+                    if chunks_received == 0:
+                        error = "No streaming chunks received"
+                else:
+                    text = await response.text()
+                    error = f"HTTP {response.status}: {text[:100]}"
+                    
+        except Exception as e:
+            error = f"Request error: {str(e)}"
+        finally:
+            await session.close()
+            
+        response_time = time.time() - start_time
+        return response_time, chunks_received, ttft, error
+
+
+    async def run_benchmark(self, duration: int, concurrent_users: int) -> BenchmarkStats:
+        """Run benchmark using async requests for specified duration."""
+        stats = BenchmarkStats()
+        stats.concurrent_users = concurrent_users
+        stats.start_time = time.time()
+        
+        print(f"Starting benchmark: {duration}s duration, {concurrent_users} concurrent users")
+        print(f"Target URL: {self.base_url}/chat/completions")
+        print(f"Model: {self.model_id}")
+        
+        connector = aiohttp.TCPConnector(limit=concurrent_users)
+        async with aiohttp.ClientSession(connector=connector) as session:
+            
+            async def worker(worker_id: int):
+                """Worker that sends requests sequentially until canceled."""
+                request_count = 0
+                while True:
+                    try:
+                        response_time, chunks, ttft, error = await self.make_async_streaming_request()
+                        await stats.add_result(response_time, chunks, ttft, error)
+                        request_count += 1
+                        
+                    except asyncio.CancelledError:
+                        break
+                    except Exception as e:
+                        await stats.add_result(0, 0, None, f"Worker {worker_id} error: {str(e)}")
+            
+            # Progress reporting task
+            async def progress_reporter():
+                last_report_time = time.time()
+                while True:
+                    try:
+                        await asyncio.sleep(1)  # Report every second
+                        if time.time() >= last_report_time + 10:  # Report every 10 seconds
+                            elapsed = time.time() - stats.start_time
+                            print(f"Completed: {stats.total_requests} requests in {elapsed:.1f}s")
+                            last_report_time = time.time()
+                    except asyncio.CancelledError:
+                        break
+            
+            # Spawn concurrent workers
+            tasks = [asyncio.create_task(worker(i)) for i in range(concurrent_users)]
+            progress_task = asyncio.create_task(progress_reporter())
+            tasks.append(progress_task)
+            
+            # Wait for duration then cancel all tasks
+            await asyncio.sleep(duration)
+            
+            for task in tasks:
+                task.cancel()
+            
+            # Wait for all tasks to complete
+            await asyncio.gather(*tasks, return_exceptions=True)
+        
+        stats.end_time = time.time()
+        return stats
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Llama Stack Benchmark Tool")
+    parser.add_argument("--base-url", default=os.getenv("BENCHMARK_BASE_URL", "http://localhost:8000/v1/openai/v1"),
+                       help="Base URL for the API (default: http://localhost:8000/v1/openai/v1)")
+    parser.add_argument("--model", default=os.getenv("INFERENCE_MODEL", "test-model"),
+                       help="Model ID to use for requests")
+    parser.add_argument("--duration", type=int, default=60,
+                       help="Duration in seconds to run benchmark (default: 60)")
+    parser.add_argument("--concurrent", type=int, default=10,
+                       help="Number of concurrent users (default: 10)")
+    
+    args = parser.parse_args()
+    
+    benchmark = LlamaStackBenchmark(args.base_url, args.model)
+    
+    try:
+        stats = asyncio.run(benchmark.run_benchmark(args.duration, args.concurrent))
+        stats.print_summary()
+        
+    except KeyboardInterrupt:
+        print("\nBenchmark interrupted by user")
+    except Exception as e:
+        print(f"Benchmark failed: {e}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/docs/source/distributions/k8s-benchmark/locust-k8s.yaml b/docs/source/distributions/k8s-benchmark/locust-k8s.yaml
deleted file mode 100644
index f20a01b2d..000000000
--- a/docs/source/distributions/k8s-benchmark/locust-k8s.yaml
+++ /dev/null
@@ -1,131 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: locust-master
-  labels:
-    app: locust
-    role: master
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: locust
-      role: master
-  template:
-    metadata:
-      labels:
-        app: locust
-        role: master
-    spec:
-      containers:
-      - name: locust-master
-        image: locustio/locust:2.31.8
-        ports:
-        - containerPort: 8089  # Web UI
-        - containerPort: 5557  # Master communication
-        env:
-        - name: LOCUST_HOST
-          value: "${LOCUST_HOST}"
-        - name: LOCUST_LOCUSTFILE
-          value: "/locust/locustfile.py"
-        - name: LOCUST_WEB_HOST
-          value: "0.0.0.0"
-        - name: LOCUST_MASTER
-          value: "true"
-        - name: LOCUST_BASE_PATH
-          value: "${LOCUST_BASE_PATH}"
-        - name: INFERENCE_MODEL
-          value: "${BENCHMARK_INFERENCE_MODEL}"
-        volumeMounts:
-        - name: locust-script
-          mountPath: /locust
-        command: ["locust"]
-        args:
-        - "--master"
-        - "--web-host=0.0.0.0"
-        - "--web-port=8089"
-        - "--host=${LOCUST_HOST}"
-        - "--locustfile=/locust/locustfile.py"
-      volumes:
-      - name: locust-script
-        configMap:
-          name: locust-script
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: locust-worker
-  labels:
-    app: locust
-    role: worker
-spec:
-  replicas: 2  # Start with 2 workers, can be scaled up
-  selector:
-    matchLabels:
-      app: locust
-      role: worker
-  template:
-    metadata:
-      labels:
-        app: locust
-        role: worker
-    spec:
-      containers:
-      - name: locust-worker
-        image: locustio/locust:2.31.8
-        env:
-        - name: LOCUST_HOST
-          value: "${LOCUST_HOST}"
-        - name: LOCUST_LOCUSTFILE
-          value: "/locust/locustfile.py"
-        - name: LOCUST_MASTER_HOST
-          value: "locust-master-service"
-        - name: LOCUST_MASTER_PORT
-          value: "5557"
-        - name: INFERENCE_MODEL
-          value: "${BENCHMARK_INFERENCE_MODEL}"
-        - name: LOCUST_BASE_PATH
-          value: "${LOCUST_BASE_PATH}"
-        volumeMounts:
-        - name: locust-script
-          mountPath: /locust
-        command: ["locust"]
-        args:
-        - "--worker"
-        - "--master-host=locust-master-service"
-        - "--master-port=5557"
-        - "--locustfile=/locust/locustfile.py"
-      volumes:
-      - name: locust-script
-        configMap:
-          name: locust-script
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: locust-master-service
-spec:
-  selector:
-    app: locust
-    role: master
-  ports:
-  - name: web-ui
-    port: 8089
-    targetPort: 8089
-  - name: master-comm
-    port: 5557
-    targetPort: 5557
-  type: ClusterIP
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: locust-web-ui
-spec:
-  selector:
-    app: locust
-    role: master
-  ports:
-  - port: 8089
-    targetPort: 8089
-  type: ClusterIP  # Keep internal, use port-forward to access
diff --git a/docs/source/distributions/k8s-benchmark/locustfile.py b/docs/source/distributions/k8s-benchmark/locustfile.py
deleted file mode 100644
index 8e511fa95..000000000
--- a/docs/source/distributions/k8s-benchmark/locustfile.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-"""
-Locust load testing script for Llama Stack with Prism mock OpenAI provider.
-"""
-
-import random
-from locust import HttpUser, task, between
-import os
-
-base_path = os.getenv("LOCUST_BASE_PATH", "/v1/openai/v1")
-
-MODEL_ID = os.getenv("INFERENCE_MODEL")
-
-class LlamaStackUser(HttpUser):
-    wait_time = between(0.0, 0.0001)
-    
-    def on_start(self):
-        """Setup authentication and test data."""
-        # No auth required for benchmark server
-        self.headers = {
-            "Content-Type": "application/json"
-        }
-        
-        # Test messages of varying lengths
-        self.test_messages = [
-            [{"role": "user", "content": "Hi"}],
-            [{"role": "user", "content": "What is the capital of France?"}],
-            [{"role": "user", "content": "Explain quantum physics in simple terms."}],
-            [{"role": "user", "content": "Write a short story about a robot learning to paint."}],
-            [
-                {"role": "user", "content": "What is machine learning?"},
-                {"role": "assistant", "content": "Machine learning is a subset of AI..."},
-                {"role": "user", "content": "Can you give me a practical example?"}
-            ]
-        ]
-
-    @task(weight=100)
-    def chat_completion_streaming(self):
-        """Test streaming chat completion (20% of requests)."""
-        messages = random.choice(self.test_messages)
-        payload = {
-            "model": MODEL_ID, 
-            "messages": messages,
-            "stream": True,
-            "max_tokens": 100
-        }
-        
-        with self.client.post(
-            f"{base_path}/chat/completions",
-            headers=self.headers,
-            json=payload,
-            stream=True,
-            catch_response=True
-        ) as response:
-            if response.status_code == 200:
-                chunks_received = 0
-                try:
-                    for line in response.iter_lines():
-                        if line:
-                            line_str = line.decode('utf-8')
-                            if line_str.startswith('data: '):
-                                chunks_received += 1
-                                if line_str.strip() == 'data: [DONE]':
-                                    break
-                    
-                    if chunks_received > 0:
-                        response.success()
-                    else:
-                        response.failure("No streaming chunks received")
-                except Exception as e:
-                    response.failure(f"Streaming error: {e}")
-            else:
-                response.failure(f"HTTP {response.status_code}: {response.text}")
diff --git a/docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml b/docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml
deleted file mode 100644
index c72921281..000000000
--- a/docs/source/distributions/k8s-benchmark/openai-mock-deployment.yaml
+++ /dev/null
@@ -1,52 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: openai-mock
-  labels:
-    app: openai-mock
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: openai-mock
-  template:
-    metadata:
-      labels:
-        app: openai-mock
-    spec:
-      containers:
-      - name: openai-mock
-        image: python:3.12-slim
-        ports:
-        - containerPort: ${MOCK_INFERENCE_PORT}
-        env:
-        - name: PORT
-          value: "${MOCK_INFERENCE_PORT}"
-        - name: MOCK_MODELS
-          value: "${MOCK_INFERENCE_MODEL}"
-        - name: STREAM_DELAY_SECONDS
-          value: "${STREAM_DELAY_SECONDS}"
-        command: ["sh", "-c"]
-        args:
-        - |
-          pip install flask &&
-          python /app/openai-mock-server.py --port ${MOCK_INFERENCE_PORT}
-        volumeMounts:
-        - name: openai-mock-script
-          mountPath: /app
-      volumes:
-      - name: openai-mock-script
-        configMap:
-          name: openai-mock
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: openai-mock-service
-spec:
-  selector:
-    app: openai-mock
-  ports:
-  - port: 8080
-    targetPort: 8080
-  type: ClusterIP
diff --git a/docs/source/distributions/k8s-benchmark/openai-mock-server.py b/docs/source/distributions/k8s-benchmark/openai-mock-server.py
index 46c923b60..de0680842 100755
--- a/docs/source/distributions/k8s-benchmark/openai-mock-server.py
+++ b/docs/source/distributions/k8s-benchmark/openai-mock-server.py
@@ -23,7 +23,7 @@ app = Flask(__name__)
 
 # Models from environment variables
 def get_models():
-    models_str = os.getenv("MOCK_MODELS", "mock-inference")
+    models_str = os.getenv("MOCK_MODELS", "meta-llama/Llama-3.2-3B-Instruct")
     model_ids = [m.strip() for m in models_str.split(",") if m.strip()]
     
     return {
@@ -49,13 +49,13 @@ def generate_random_text(length=50):
     ]
     return " ".join(random.choices(words, k=length))
 
-@app.route('/models', methods=['GET'])
+@app.route('/v1/models', methods=['GET'])
 def list_models():
     models = get_models()
     print(f"[MOCK] Returning models: {[m['id'] for m in models['data']]}")
     return jsonify(models)
 
-@app.route('/chat/completions', methods=['POST'])
+@app.route('/v1/chat/completions', methods=['POST'])
 def chat_completions():
     """Return OpenAI-formatted chat completion responses."""
     data = request.get_json()
diff --git a/docs/source/distributions/k8s-benchmark/profile_running_server.sh b/docs/source/distributions/k8s-benchmark/profile_running_server.sh
new file mode 100755
index 000000000..65d620583
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/profile_running_server.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+# Script to profile an already running Llama Stack server
+# Usage: ./profile_running_server.sh [duration_seconds] [output_file]
+
+DURATION=${1:-60}  # Default 60 seconds
+OUTPUT_FILE=${2:-"llama_stack_profile"}  # Default output file
+
+echo "Looking for running Llama Stack server..."
+
+# Find the server PID
+SERVER_PID=$(ps aux | grep "llama_stack.core.server.server" | grep -v grep | awk '{print $2}' | head -1)
+
+
+if [ -z "$SERVER_PID" ]; then
+    echo "Error: No running Llama Stack server found"
+    echo "Please start your server first with:"
+    echo "LLAMA_STACK_LOGGING=\"all=ERROR\" MOCK_INFERENCE_URL=http://localhost:8080 SAFETY_MODEL=llama-guard3:1b uv run --with llama-stack python -m llama_stack.core.server.server docs/source/distributions/k8s-benchmark/stack_run_config.yaml"
+    exit 1
+fi
+
+echo "Found Llama Stack server with PID: $SERVER_PID"
+
+# Start py-spy profiling
+echo "Starting py-spy profiling for ${DURATION} seconds..."
+echo "Output will be saved to: ${OUTPUT_FILE}.svg"
+echo ""
+echo "You can now run your load test..."
+echo ""
+
+# Get the full path to py-spy
+PYSPY_PATH=$(which py-spy)
+
+# Check if running as root, if not, use sudo
+if [ "$EUID" -ne 0 ]; then
+    echo "py-spy requires root permissions on macOS. Running with sudo..."
+    sudo "$PYSPY_PATH" record -o "${OUTPUT_FILE}.svg" -d ${DURATION} -p $SERVER_PID
+else
+    "$PYSPY_PATH" record -o "${OUTPUT_FILE}.svg" -d ${DURATION} -p $SERVER_PID
+fi
+
+echo ""
+echo "Profiling completed! Results saved to: ${OUTPUT_FILE}.svg"
+echo ""
+echo "To view the flame graph:"
+echo "open ${OUTPUT_FILE}.svg"
diff --git a/docs/source/distributions/k8s-benchmark/run-benchmark.sh b/docs/source/distributions/k8s-benchmark/run-benchmark.sh
new file mode 100755
index 000000000..e1c826143
--- /dev/null
+++ b/docs/source/distributions/k8s-benchmark/run-benchmark.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+set -euo pipefail
+
+# Default values
+TARGET="stack"
+DURATION=60
+CONCURRENT=10
+
+# Parse command line arguments
+usage() {
+    echo "Usage: $0 [options]"
+    echo "Options:"
+    echo "  -t, --target <stack|vllm>     Target to benchmark (default: stack)"
+    echo "  -d, --duration <seconds>      Duration in seconds (default: 60)"
+    echo "  -c, --concurrent <users>      Number of concurrent users (default: 10)"
+    echo "  -h, --help                    Show this help message"
+    echo ""
+    echo "Examples:"
+    echo "  $0 --target vllm              # Benchmark vLLM direct"
+    echo "  $0 --target stack             # Benchmark Llama Stack (default)"
+    echo "  $0 -t vllm -d 120 -c 20       # vLLM with 120s duration, 20 users"
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -t|--target)
+            TARGET="$2"
+            shift 2
+            ;;
+        -d|--duration)
+            DURATION="$2"
+            shift 2
+            ;;
+        -c|--concurrent)
+            CONCURRENT="$2"
+            shift 2
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+# Validate target
+if [[ "$TARGET" != "stack" && "$TARGET" != "vllm" ]]; then
+    echo "Error: Target must be 'stack' or 'vllm'"
+    usage
+    exit 1
+fi
+
+# Set configuration based on target
+if [[ "$TARGET" == "vllm" ]]; then
+    BASE_URL="http://vllm-server:8000/v1"
+    JOB_NAME="vllm-benchmark-job"
+    echo "Benchmarking vLLM direct..."
+else
+    BASE_URL="http://llama-stack-benchmark-service:8323/v1/openai/v1"
+    JOB_NAME="stack-benchmark-job"
+    echo "Benchmarking Llama Stack..."
+fi
+
+echo "Configuration:"
+echo "  Target: $TARGET"
+echo "  Base URL: $BASE_URL"
+echo "  Duration: ${DURATION}s"
+echo "  Concurrent users: $CONCURRENT"
+echo ""
+
+# Create temporary job yaml
+TEMP_YAML="/tmp/benchmark-job-temp-$(date +%s).yaml"
+cat > "$TEMP_YAML" << EOF
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: $JOB_NAME
+  namespace: default
+spec:
+  template:
+    spec:
+      containers:
+      - name: benchmark
+        image: python:3.11-slim
+        command: ["/bin/bash"]
+        args:
+        - "-c"
+        - |
+          pip install aiohttp &&
+          python3 /benchmark/benchmark.py \\
+            --base-url $BASE_URL \\
+            --model \${INFERENCE_MODEL} \\
+            --duration $DURATION \\
+            --concurrent $CONCURRENT
+        env:
+        - name: INFERENCE_MODEL
+          value: "meta-llama/Llama-3.2-3B-Instruct"
+        volumeMounts:
+        - name: benchmark-script
+          mountPath: /benchmark
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "250m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+      volumes:
+      - name: benchmark-script
+        configMap:
+          name: benchmark-script
+      restartPolicy: Never
+  backoffLimit: 3
+EOF
+
+echo "Creating benchmark ConfigMap..."
+kubectl create configmap benchmark-script \
+  --from-file=benchmark.py=benchmark.py \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Cleaning up any existing benchmark job..."
+kubectl delete job $JOB_NAME 2>/dev/null || true
+
+echo "Deploying benchmark Job..."
+kubectl apply -f "$TEMP_YAML"
+
+echo "Waiting for job to start..."
+kubectl wait --for=condition=Ready pod -l job-name=$JOB_NAME --timeout=60s
+
+echo "Following benchmark logs..."
+kubectl logs -f job/$JOB_NAME
+
+echo "Job completed. Checking final status..."
+kubectl get job $JOB_NAME
+
+# Clean up temporary file
+rm -f "$TEMP_YAML"
diff --git a/docs/source/distributions/k8s-benchmark/stack-configmap.yaml b/docs/source/distributions/k8s-benchmark/stack-configmap.yaml
index 653e66756..edf4ebd75 100644
--- a/docs/source/distributions/k8s-benchmark/stack-configmap.yaml
+++ b/docs/source/distributions/k8s-benchmark/stack-configmap.yaml
@@ -26,13 +26,6 @@ data:
           max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
           api_token: ${env.VLLM_API_TOKEN:=fake}
           tls_verify: ${env.VLLM_TLS_VERIFY:=true}
-      - provider_id: mock-vllm-inference
-        provider_type: remote::vllm
-        config:
-          url: http://openai-mock-service:${env.MOCK_INFERENCE_PORT}
-          max_tokens: 4096
-          api_token: fake
-          tls_verify: false
       - provider_id: sentence-transformers
         provider_type: inline::sentence-transformers
         config: {}
@@ -121,9 +114,6 @@ data:
     - model_id: ${env.SAFETY_MODEL}
       provider_id: vllm-safety
       model_type: llm
-    - model_id: ${env.MOCK_INFERENCE_MODEL}
-      provider_id: mock-vllm-inference
-      model_type: llm
     shields:
     - shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
     vector_dbs: []
diff --git a/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template b/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template
index bc14d5124..9cb1e5be3 100644
--- a/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template
+++ b/docs/source/distributions/k8s-benchmark/stack-k8s.yaml.template
@@ -44,8 +44,6 @@ spec:
           value: "${SAFETY_MODEL}"
         - name: TAVILY_SEARCH_API_KEY
           value: "${TAVILY_SEARCH_API_KEY}"
-        - name: MOCK_INFERENCE_PORT
-          value: "${MOCK_INFERENCE_PORT}"
         - name: VLLM_URL
           value: http://vllm-server.default.svc.cluster.local:8000/v1
         - name: VLLM_MAX_TOKENS
@@ -54,8 +52,6 @@ spec:
           value: http://vllm-server-safety.default.svc.cluster.local:8001/v1
         - name: VLLM_TLS_VERIFY
           value: "false"
-        - name: MOCK_INFERENCE_MODEL
-          value: "${MOCK_INFERENCE_MODEL}"
         command: ["python", "-m", "llama_stack.core.server.server", "/etc/config/stack_run_config.yaml", "--port", "8323"]
         ports:
           - containerPort: 8323
diff --git a/docs/source/distributions/k8s-benchmark/stack_run_config.yaml b/docs/source/distributions/k8s-benchmark/stack_run_config.yaml
index ad56be047..ceb1ba2d9 100644
--- a/docs/source/distributions/k8s-benchmark/stack_run_config.yaml
+++ b/docs/source/distributions/k8s-benchmark/stack_run_config.yaml
@@ -3,7 +3,6 @@ image_name: kubernetes-benchmark-demo
 apis:
 - agents
 - inference
-- safety
 - telemetry
 - tool_runtime
 - vector_io
@@ -16,20 +15,6 @@ providers:
       max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
       api_token: ${env.VLLM_API_TOKEN:=fake}
       tls_verify: ${env.VLLM_TLS_VERIFY:=true}
-  - provider_id: vllm-safety
-    provider_type: remote::vllm
-    config:
-      url: ${env.VLLM_SAFETY_URL:=http://localhost:8000/v1}
-      max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
-      api_token: ${env.VLLM_API_TOKEN:=fake}
-      tls_verify: ${env.VLLM_TLS_VERIFY:=true}
-  - provider_id: mock-vllm-inference
-    provider_type: remote::vllm
-    config:
-      url: http://openai-mock-service:${env.MOCK_INFERENCE_PORT}
-      max_tokens: 4096
-      api_token: fake
-      tls_verify: false
   - provider_id: sentence-transformers
     provider_type: inline::sentence-transformers
     config: {}
@@ -45,11 +30,6 @@ providers:
         db: ${env.POSTGRES_DB:=llamastack}
         user: ${env.POSTGRES_USER:=llamastack}
         password: ${env.POSTGRES_PASSWORD:=llamastack}
-  safety:
-  - provider_id: llama-guard
-    provider_type: inline::llama-guard
-    config:
-      excluded_categories: []
   agents:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
@@ -115,14 +95,6 @@ models:
 - model_id: ${env.INFERENCE_MODEL}
   provider_id: vllm-inference
   model_type: llm
-- model_id: ${env.SAFETY_MODEL}
-  provider_id: vllm-safety
-  model_type: llm
-- model_id: ${env.MOCK_INFERENCE_MODEL}
-  provider_id: mock-vllm-inference
-  model_type: llm
-shields:
-- shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
 vector_dbs: []
 datasets: []
 scoring_fns: []

From a6e2c1890932fbf0514952038482c2ce5f0d5c75 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 12:01:07 -0700
Subject: [PATCH 41/45] Revert "refactor(agents): migrate to OpenAI chat
 completions API" (#3167)

Reverts llamastack/llama-stack#3097

It has broken agents tests.
---
 .../agents/meta_reference/agent_instance.py   | 67 +++----------------
 1 file changed, 9 insertions(+), 58 deletions(-)

diff --git a/llama_stack/providers/inline/agents/meta_reference/agent_instance.py b/llama_stack/providers/inline/agents/meta_reference/agent_instance.py
index e9f89f8d2..5f7c90879 100644
--- a/llama_stack/providers/inline/agents/meta_reference/agent_instance.py
+++ b/llama_stack/providers/inline/agents/meta_reference/agent_instance.py
@@ -68,11 +68,6 @@ from llama_stack.models.llama.datatypes import (
     BuiltinTool,
     ToolCall,
 )
-from llama_stack.providers.utils.inference.openai_compat import (
-    convert_message_to_openai_dict,
-    convert_openai_chat_completion_stream,
-    convert_tooldef_to_openai_tool,
-)
 from llama_stack.providers.utils.kvstore import KVStore
 from llama_stack.providers.utils.telemetry import tracing
 
@@ -515,60 +510,16 @@ class ChatAgent(ShieldRunnerMixin):
             async with tracing.span("inference") as span:
                 if self.agent_config.name:
                     span.set_attribute("agent_name", self.agent_config.name)
-                # Convert messages to OpenAI format
-                openai_messages = []
-                for message in input_messages:
-                    openai_message = await convert_message_to_openai_dict(message)
-                    openai_messages.append(openai_message)
-
-                # Convert tool definitions to OpenAI format
-                openai_tools = None
-                if self.tool_defs:
-                    openai_tools = []
-                    for tool_def in self.tool_defs:
-                        openai_tool = convert_tooldef_to_openai_tool(tool_def)
-                        openai_tools.append(openai_tool)
-
-                # Extract tool_choice from tool_config for OpenAI compatibility
-                # Note: tool_choice can only be provided when tools are also provided
-                tool_choice = None
-                if openai_tools and self.agent_config.tool_config and self.agent_config.tool_config.tool_choice:
-                    tool_choice = (
-                        self.agent_config.tool_config.tool_choice.value
-                        if hasattr(self.agent_config.tool_config.tool_choice, "value")
-                        else str(self.agent_config.tool_config.tool_choice)
-                    )
-
-                # Convert sampling params to OpenAI format (temperature, top_p, max_tokens)
-                temperature = None
-                top_p = None
-                max_tokens = None
-                if sampling_params:
-                    if hasattr(sampling_params.strategy, "temperature"):
-                        temperature = sampling_params.strategy.temperature
-                    if hasattr(sampling_params.strategy, "top_p"):
-                        top_p = sampling_params.strategy.top_p
-                    if sampling_params.max_tokens:
-                        max_tokens = sampling_params.max_tokens
-
-                # Use OpenAI chat completion
-                openai_stream = await self.inference_api.openai_chat_completion(
-                    model=self.agent_config.model,
-                    messages=openai_messages,
-                    tools=openai_tools if openai_tools else None,
-                    tool_choice=tool_choice,
-                    temperature=temperature,
-                    top_p=top_p,
-                    max_tokens=max_tokens,
+                async for chunk in await self.inference_api.chat_completion(
+                    self.agent_config.model,
+                    input_messages,
+                    tools=self.tool_defs,
+                    tool_prompt_format=self.agent_config.tool_config.tool_prompt_format,
+                    response_format=self.agent_config.response_format,
                     stream=True,
-                )
-
-                # Convert OpenAI stream back to Llama Stack format
-                response_stream = convert_openai_chat_completion_stream(
-                    openai_stream, enable_incremental_tool_calls=True
-                )
-
-                async for chunk in response_stream:
+                    sampling_params=sampling_params,
+                    tool_config=self.agent_config.tool_config,
+                ):
                     event = chunk.event
                     if event.event_type == ChatCompletionResponseEventType.start:
                         continue

From 0e8bb94bf3100857e2bd2b3cc910992896ca88b0 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 14:47:20 -0700
Subject: [PATCH 42/45] feat(ci): make recording workflow simpler, more
 parameterizable (#3169)

# What does this PR do?

Recording tests has become a nightmare. This is the first part of making
that process simpler by making it _less_ automatic. I tried to be too
clever earlier.

It simplifies the record-integration-tests workflow to use workflow
dispatch inputs instead of PR labels. No more opaque stuff. Just go to
the GitHub UI and run the workflow with inputs. I will soon add a helper
script for this also.

Other things to aid re-running just the small set of things you need to
re-record:
- Replaces the `test-types` JSON array parameter with a more intuitive
`test-subdirs` comma-separated list. The whole JSON array crap was for
matrix.
- Adds a new `test-pattern` parameter to allow filtering tests using
pytest's `-k` option


## Test Plan

Note that this PR is in a fork not the source repository.

- Replay tests on this PR are green
- Manually
[ran](https://github.com/ashwinb/llama-stack/actions/runs/16998562926)
the replay workflow with a test-subdir and test-pattern filter, worked
- Manually
[ran](https://github.com/ashwinb/llama-stack/actions/runs/16998556104/job/48195080344)
the **record** workflow with a simple pattern, it has worked and updated
_this_ PR.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
---
 .../actions/run-and-record-tests/action.yml   |  20 ++--
 .github/workflows/integration-tests.yml       |  31 ++----
 .../workflows/record-integration-tests.yml    |  91 ++++--------------
 scripts/integration-tests.sh                  |  80 ++++++++++-----
 tests/integration/recordings/index.sqlite     | Bin 53248 -> 53248 bytes
 .../recordings/responses/4a3a4447b16b.json    |  88 +----------------
 .../recordings/responses/561746e1c8de.json    |  30 +++---
 7 files changed, 119 insertions(+), 221 deletions(-)

diff --git a/.github/actions/run-and-record-tests/action.yml b/.github/actions/run-and-record-tests/action.yml
index 573148e46..1406c6077 100644
--- a/.github/actions/run-and-record-tests/action.yml
+++ b/.github/actions/run-and-record-tests/action.yml
@@ -2,9 +2,13 @@ name: 'Run and Record Tests'
 description: 'Run integration tests and handle recording/artifact upload'
 
 inputs:
-  test-types:
-    description: 'JSON array of test types to run'
+  test-subdirs:
+    description: 'Comma-separated list of test subdirectories to run'
     required: true
+  test-pattern:
+    description: 'Regex pattern to pass to pytest -k'
+    required: false
+    default: ''
   stack-config:
     description: 'Stack configuration to use'
     required: true
@@ -35,9 +39,11 @@ runs:
         ./scripts/integration-tests.sh \
           --stack-config '${{ inputs.stack-config }}' \
           --provider '${{ inputs.provider }}' \
-          --test-types '${{ inputs.test-types }}' \
+          --test-subdirs '${{ inputs.test-subdirs }}' \
+          --test-pattern '${{ inputs.test-pattern }}' \
           --inference-mode '${{ inputs.inference-mode }}' \
-          ${{ inputs.run-vision-tests == 'true' && '--run-vision-tests' || '' }}
+          ${{ inputs.run-vision-tests == 'true' && '--run-vision-tests' || '' }} \
+          | tee pytest-${{ inputs.inference-mode }}.log
 
 
     - name: Commit and push recordings
@@ -57,10 +63,10 @@ runs:
             git commit -m "Recordings update from CI"
           fi
 
-          git fetch origin ${{ github.event.pull_request.head.ref }}
-          git rebase origin/${{ github.event.pull_request.head.ref }}
+          git fetch origin ${{ github.ref_name }}
+          git rebase origin/${{ github.ref_name }}
           echo "Rebased successfully"
-          git push origin HEAD:${{ github.event.pull_request.head.ref }}
+          git push origin HEAD:${{ github.ref_name }}
           echo "Pushed successfully"
         else
           echo "No recording changes"
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 9ef49fba3..fc56f62ea 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -31,6 +31,14 @@ on:
         description: 'Test against a specific provider'
         type: string
         default: 'ollama'
+      test-subdirs:
+        description: 'Comma-separated list of test subdirectories to run'
+        type: string
+        default: ''
+      test-pattern:
+        description: 'Regex pattern to pass to pytest -k'
+        type: string
+        default: ''
 
 concurrency:
   # Skip concurrency for pushes to main - each commit should be tested independently
@@ -38,28 +46,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  discover-tests:
-    runs-on: ubuntu-latest
-    outputs:
-      test-types: ${{ steps.generate-test-types.outputs.test-types }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Generate test types
-        id: generate-test-types
-        run: |
-          # Get test directories dynamically, excluding non-test directories
-          # NOTE: we are excluding post_training since the tests take too long
-          TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d |
-            sed 's|tests/integration/||' |
-            grep -Ev "^(__pycache__|fixtures|test_cases|recordings|non_ci|post_training)$" |
-            sort | jq -R -s -c 'split("\n")[:-1]')
-          echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
 
   run-replay-mode-tests:
-    needs: discover-tests
     runs-on: ubuntu-latest
     name: ${{ format('Integration Tests ({0}, {1}, {2}, client={3}, vision={4})', matrix.client-type, matrix.provider, matrix.python-version, matrix.client-version, matrix.run-vision-tests) }}
 
@@ -90,7 +78,8 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/run-and-record-tests
         with:
-          test-types: ${{ needs.discover-tests.outputs.test-types }}
+          test-subdirs: ${{ inputs.test-subdirs }}
+          test-pattern: ${{ inputs.test-pattern }}
           stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }}
           provider: ${{ matrix.provider }}
           inference-mode: 'replay'
diff --git a/.github/workflows/record-integration-tests.yml b/.github/workflows/record-integration-tests.yml
index b31709a4f..95403291c 100644
--- a/.github/workflows/record-integration-tests.yml
+++ b/.github/workflows/record-integration-tests.yml
@@ -1,93 +1,43 @@
+# This workflow should be run manually when needing to re-record tests. This happens when you have
+#  - added a new test
+#  - or changed an existing test such that a new inference call is made
+# You should make a PR and then run this workflow on that PR branch. The workflow will re-record the
+# tests and commit the recordings to the PR branch.
 name: Integration Tests (Record)
 
 run-name: Run the integration test suite from tests/integration
 
 on:
-  pull_request_target:
-    branches: [ main ]
-    types: [opened, synchronize, labeled]
-    paths:
-      - 'llama_stack/**'
-      - 'tests/**'
-      - 'uv.lock'
-      - 'pyproject.toml'
-      - '.github/workflows/record-integration-tests.yml' # This workflow
-      - '.github/actions/setup-ollama/action.yml'
-      - '.github/actions/setup-test-environment/action.yml'
-      - '.github/actions/run-and-record-tests/action.yml'
   workflow_dispatch:
     inputs:
+      test-subdirs:
+        description: 'Comma-separated list of test subdirectories to run'
+        type: string
+        default: ''
       test-provider:
         description: 'Test against a specific provider'
         type: string
         default: 'ollama'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
+      run-vision-tests:
+        description: 'Whether to run vision tests'
+        type: boolean
+        default: false
+      test-pattern:
+        description: 'Regex pattern to pass to pytest -k'
+        type: string
+        default: ''
 
 jobs:
-  discover-tests:
-    if: contains(github.event.pull_request.labels.*.name, 're-record-tests') ||
-      contains(github.event.pull_request.labels.*.name, 're-record-vision-tests')
-    runs-on: ubuntu-latest
-    outputs:
-      test-types: ${{ steps.generate-test-types.outputs.test-types }}
-      matrix-modes: ${{ steps.generate-test-types.outputs.matrix-modes }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Generate test types
-        id: generate-test-types
-        run: |
-          # Get test directories dynamically, excluding non-test directories
-          TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
-            grep -Ev "^(__pycache__|fixtures|test_cases|recordings|post_training)$" |
-            sort | jq -R -s -c 'split("\n")[:-1]')
-          echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
-
-          labels=$(gh pr view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name')
-          echo "labels=$labels"
-
-          modes_array=()
-          if [[ $labels == *"re-record-vision-tests"* ]]; then
-            modes_array+=("vision")
-          fi
-          if [[ $labels == *"re-record-tests"* ]]; then
-            modes_array+=("non-vision")
-          fi
-
-          # Convert to JSON array
-          if [ ${#modes_array[@]} -eq 0 ]; then
-            matrix_modes="[]"
-          else
-            matrix_modes=$(printf '%s\n' "${modes_array[@]}" | jq -R -s -c 'split("\n")[:-1]')
-          fi
-          echo "matrix_modes=$matrix_modes"
-          echo "matrix-modes=$matrix_modes" >> $GITHUB_OUTPUT
-
-        env:
-          GH_TOKEN: ${{ github.token }}
-
   record-tests:
-    needs: discover-tests
     runs-on: ubuntu-latest
 
     permissions:
       contents: write
 
-    strategy:
-      fail-fast: false
-      matrix:
-        mode: ${{ fromJSON(needs.discover-tests.outputs.matrix-modes) }}
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ github.event.pull_request.head.ref }}
           fetch-depth: 0
 
       - name: Setup test environment
@@ -96,14 +46,15 @@ jobs:
           python-version: "3.12"  # Use single Python version for recording
           client-version: "latest"
           provider: ${{ inputs.test-provider || 'ollama' }}
-          run-vision-tests: ${{ matrix.mode == 'vision' && 'true' || 'false' }}
+          run-vision-tests: ${{ inputs.run-vision-tests }}
           inference-mode: 'record'
 
       - name: Run and record tests
         uses: ./.github/actions/run-and-record-tests
         with:
-          test-types: ${{ needs.discover-tests.outputs.test-types }}
+          test-pattern: ${{ inputs.test-pattern }}
+          test-subdirs: ${{ inputs.test-subdirs }}
           stack-config: 'server:ci-tests'  # recording must be done with server since more tests are run
           provider: ${{ inputs.test-provider || 'ollama' }}
           inference-mode: 'record'
-          run-vision-tests: ${{ matrix.mode == 'vision' && 'true' || 'false' }}
+          run-vision-tests: ${{ inputs.run-vision-tests }}
diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh
index 8dbbcae90..95b78e271 100755
--- a/scripts/integration-tests.sh
+++ b/scripts/integration-tests.sh
@@ -14,7 +14,8 @@ set -euo pipefail
 # Default values
 STACK_CONFIG=""
 PROVIDER=""
-TEST_TYPES='["inference"]'
+TEST_SUBDIRS=""
+TEST_PATTERN=""
 RUN_VISION_TESTS="false"
 INFERENCE_MODE="replay"
 EXTRA_PARAMS=""
@@ -27,23 +28,24 @@ Usage: $0 [OPTIONS]
 Options:
     --stack-config STRING    Stack configuration to use (required)
     --provider STRING        Provider to use (ollama, vllm, etc.) (required)
-    --test-types JSON        JSON array of test types to run (default: '["inference"]')
+    --test-subdirs STRING    Comma-separated list of test subdirectories to run (default: 'inference')
     --run-vision-tests       Run vision tests instead of regular tests
     --inference-mode STRING  Inference mode: record or replay (default: replay)
+    --test-pattern STRING    Regex pattern to pass to pytest -k
     --help                   Show this help message
 
 Examples:
     # Basic inference tests with ollama
-    $0 --stack-config server:ollama --provider ollama
+    $0 --stack-config server:ci-tests --provider ollama
 
-    # Multiple test types with vllm
-    $0 --stack-config server:vllm --provider vllm --test-types '["inference", "agents"]'
+    # Multiple test directories with vllm
+    $0 --stack-config server:ci-tests --provider vllm --test-subdirs 'inference,agents'
 
     # Vision tests with ollama
-    $0 --stack-config server:ollama --provider ollama --run-vision-tests
+    $0 --stack-config server:ci-tests --provider ollama --run-vision-tests
 
     # Record mode for updating test recordings
-    $0 --stack-config server:ollama --provider ollama --inference-mode record
+    $0 --stack-config server:ci-tests --provider ollama --inference-mode record
 EOF
 }
 
@@ -58,8 +60,8 @@ while [[ $# -gt 0 ]]; do
             PROVIDER="$2"
             shift 2
             ;;
-        --test-types)
-            TEST_TYPES="$2"
+        --test-subdirs)
+            TEST_SUBDIRS="$2"
             shift 2
             ;;
         --run-vision-tests)
@@ -70,6 +72,10 @@ while [[ $# -gt 0 ]]; do
             INFERENCE_MODE="$2"
             shift 2
             ;;
+        --test-pattern)
+            TEST_PATTERN="$2"
+            shift 2
+            ;;
         --help)
             usage
             exit 0
@@ -99,9 +105,10 @@ fi
 echo "=== Llama Stack Integration Test Runner ==="
 echo "Stack Config: $STACK_CONFIG"
 echo "Provider: $PROVIDER"
-echo "Test Types: $TEST_TYPES"
+echo "Test Subdirs: $TEST_SUBDIRS"
 echo "Vision Tests: $RUN_VISION_TESTS"
 echo "Inference Mode: $INFERENCE_MODE"
+echo "Test Pattern: $TEST_PATTERN"
 echo ""
 
 # Check storage and memory before tests
@@ -164,17 +171,29 @@ if [[ "$PROVIDER" == "vllm" ]]; then
     EXCLUDE_TESTS="${EXCLUDE_TESTS} or test_inference_store_tool_calls"
 fi
 
+PYTEST_PATTERN="not( $EXCLUDE_TESTS )"
+if [[ -n "$TEST_PATTERN" ]]; then
+    PYTEST_PATTERN="${PYTEST_PATTERN} and $TEST_PATTERN"
+fi
+
 # Run vision tests if specified
 if [[ "$RUN_VISION_TESTS" == "true" ]]; then
     echo "Running vision tests..."
-    if uv run pytest -s -v tests/integration/inference/test_vision_inference.py \
+    set +e
+    uv run pytest -s -v tests/integration/inference/test_vision_inference.py \
         --stack-config="$STACK_CONFIG" \
-        -k "not( $EXCLUDE_TESTS )" \
+        -k "$PYTEST_PATTERN" \
         --vision-model=ollama/llama3.2-vision:11b \
         --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
         --color=yes $EXTRA_PARAMS \
-        --capture=tee-sys | tee pytest-${INFERENCE_MODE}-vision.log; then
+        --capture=tee-sys
+    exit_code=$?
+    set -e
+
+    if [ $exit_code -eq 0 ]; then
         echo "✅ Vision tests completed successfully"
+    elif [ $exit_code -eq 5 ]; then
+        echo "⚠️ No vision tests collected (pattern matched no tests)"
     else
         echo "❌ Vision tests failed"
         exit 1
@@ -183,28 +202,34 @@ if [[ "$RUN_VISION_TESTS" == "true" ]]; then
 fi
 
 # Run regular tests
-echo "Test types to run: $TEST_TYPES"
+if [[ -z "$TEST_SUBDIRS" ]]; then
+   TEST_SUBDIRS=$(find tests/integration -maxdepth 1 -mindepth 1 -type d |
+            sed 's|tests/integration/||' |
+            grep -Ev "^(__pycache__|fixtures|test_cases|recordings|non_ci|post_training)$" |
+            sort)
+fi
+echo "Test subdirs to run: $TEST_SUBDIRS"
 
 # Collect all test files for the specified test types
 TEST_FILES=""
-for test_type in $(echo "$TEST_TYPES" | jq -r '.[]'); do
+for test_subdir in $(echo "$TEST_SUBDIRS" | tr ',' '\n'); do
     # Skip certain test types for vllm provider
     if [[ "$PROVIDER" == "vllm" ]]; then
-        if [[ "$test_type" == "safety" ]] || [[ "$test_type" == "post_training" ]] || [[ "$test_type" == "tool_runtime" ]]; then
-            echo "Skipping $test_type for vllm provider"
+        if [[ "$test_subdir" == "safety" ]] || [[ "$test_subdir" == "post_training" ]] || [[ "$test_subdir" == "tool_runtime" ]]; then
+            echo "Skipping $test_subdir for vllm provider"
             continue
         fi
     fi
 
-    if [[ -d "tests/integration/$test_type" ]]; then
+    if [[ -d "tests/integration/$test_subdir" ]]; then
         # Find all Python test files in this directory
-        test_files=$(find tests/integration/$test_type -name "test_*.py" -o -name "*_test.py")
+        test_files=$(find tests/integration/$test_subdir -name "test_*.py" -o -name "*_test.py")
         if [[ -n "$test_files" ]]; then
             TEST_FILES="$TEST_FILES $test_files"
-            echo "Added test files from $test_type: $(echo $test_files | wc -w) files"
+            echo "Added test files from $test_subdir: $(echo $test_files | wc -w) files"
         fi
     else
-        echo "Warning: Directory tests/integration/$test_type does not exist"
+        echo "Warning: Directory tests/integration/$test_subdir does not exist"
     fi
 done
 
@@ -217,14 +242,21 @@ echo ""
 echo "=== Running all collected tests in a single pytest command ==="
 echo "Total test files: $(echo $TEST_FILES | wc -w)"
 
-if uv run pytest -s -v $TEST_FILES \
+set +e
+uv run pytest -s -v $TEST_FILES \
     --stack-config="$STACK_CONFIG" \
-    -k "not( $EXCLUDE_TESTS )" \
+    -k "$PYTEST_PATTERN" \
     --text-model="$TEXT_MODEL" \
     --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
     --color=yes $EXTRA_PARAMS \
-    --capture=tee-sys | tee pytest-${INFERENCE_MODE}-all.log; then
+    --capture=tee-sys
+exit_code=$?
+set -e
+
+if [ $exit_code -eq 0 ]; then
     echo "✅ All tests completed successfully"
+elif [ $exit_code -eq 5 ]; then
+    echo "⚠️ No tests collected (pattern matched no tests)"
 else
     echo "❌ Tests failed"
     exit 1
diff --git a/tests/integration/recordings/index.sqlite b/tests/integration/recordings/index.sqlite
index 1951ee7d61338c23da0e6a96cd6a585eb0adf134..e01c8803aba178128bbcf05b06a5c949fcd928b1 100644
GIT binary patch
delta 244
zcmZozz}&Ead4e>f>qHr6L01O7vQA$99}G-<oeX@P{D(IdO7Ly&;HwN|;}m4jVf3BM
zo)FH&#xgk}VcEn6_sI&!SvLnIvNN+exv@0zGEH`HbC^6o#e>m)^NSQ2#>s4nu1u^h
zo4wPHdrlVFpuofMlOcvJiR~750Z+}uLb=T`8#ve)?I%|YWC&R38k#B?8CV&aSQ#3Z
zb~#PHyDgUo#%~nraGg9ysCDxEojjYLZ|`R2bmnN})tBZq=9p}M)?xDQ-O3zLwdRwb
g?ViEZ<ubWpPdhi<NK?bjLVNQB7@apao~>sB0Bh+@2><{9

delta 275
zcmZozz}&Ead4e>f#Y7orK??@GvTk1f9}G-<oeX@P{D(IdO7Ly&;HwN|V`E{^VPu=k
zo)FFiq!JRAO;$M0I+-)gfr-Urvu`2~Gn<hcOCvASWCu5g$@5b@7_~RQNReS=;S^+W
zpIGP&QoT7K?U1JqPYw4ou3KC#oNG8mIO^D6um`bTW_4j%&cel<%XERsm~k3g65B2A
z0*0RqF%t{rHpgt>U}MytTrH3xV4-Vhs$gJjWn^Y$Xi(Z^H2LnfTplRjuu-VPb@Cjc
z*2(jC@@#&-y_=cSn4^(bUz*pLW3v5OhsnElD{}zV8k<^~SWbSndj?a7$>fGT?UUp8
Ub1|B17TTL9z-YX=@oYU40NKq{`Tzg`

diff --git a/tests/integration/recordings/responses/4a3a4447b16b.json b/tests/integration/recordings/responses/4a3a4447b16b.json
index 96b40a792..ee1ee6d70 100644
--- a/tests/integration/recordings/responses/4a3a4447b16b.json
+++ b/tests/integration/recordings/responses/4a3a4447b16b.json
@@ -14,7 +14,7 @@
         "models": [
           {
             "model": "nomic-embed-text:latest",
-            "modified_at": "2025-08-14T20:26:10.795125-07:00",
+            "modified_at": "2025-08-15T20:24:13.254634Z",
             "digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f",
             "size": 274302450,
             "details": {
@@ -28,41 +28,9 @@
               "quantization_level": "F16"
             }
           },
-          {
-            "model": "llama3.2-vision:11b",
-            "modified_at": "2025-07-30T18:45:02.517873-07:00",
-            "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e",
-            "size": 7816589186,
-            "details": {
-              "parent_model": "",
-              "format": "gguf",
-              "family": "mllama",
-              "families": [
-                "mllama"
-              ],
-              "parameter_size": "10.7B",
-              "quantization_level": "Q4_K_M"
-            }
-          },
-          {
-            "model": "llama3.2-vision:latest",
-            "modified_at": "2025-07-29T20:18:47.920468-07:00",
-            "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e",
-            "size": 7816589186,
-            "details": {
-              "parent_model": "",
-              "format": "gguf",
-              "family": "mllama",
-              "families": [
-                "mllama"
-              ],
-              "parameter_size": "10.7B",
-              "quantization_level": "Q4_K_M"
-            }
-          },
           {
             "model": "llama-guard3:1b",
-            "modified_at": "2025-07-25T14:39:44.978630-07:00",
+            "modified_at": "2025-07-31T04:44:58Z",
             "digest": "494147e06bf99e10dbe67b63a07ac81c162f18ef3341aa3390007ac828571b3b",
             "size": 1600181919,
             "details": {
@@ -78,7 +46,7 @@
           },
           {
             "model": "all-minilm:l6-v2",
-            "modified_at": "2025-07-24T15:15:11.129290-07:00",
+            "modified_at": "2025-07-31T04:42:15Z",
             "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef",
             "size": 45960996,
             "details": {
@@ -92,57 +60,9 @@
               "quantization_level": "F16"
             }
           },
-          {
-            "model": "llama3.2:1b",
-            "modified_at": "2025-07-17T22:02:24.953208-07:00",
-            "digest": "baf6a787fdffd633537aa2eb51cfd54cb93ff08e28040095462bb63daf552878",
-            "size": 1321098329,
-            "details": {
-              "parent_model": "",
-              "format": "gguf",
-              "family": "llama",
-              "families": [
-                "llama"
-              ],
-              "parameter_size": "1.2B",
-              "quantization_level": "Q8_0"
-            }
-          },
-          {
-            "model": "all-minilm:latest",
-            "modified_at": "2025-06-03T16:50:10.946583-07:00",
-            "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef",
-            "size": 45960996,
-            "details": {
-              "parent_model": "",
-              "format": "gguf",
-              "family": "bert",
-              "families": [
-                "bert"
-              ],
-              "parameter_size": "23M",
-              "quantization_level": "F16"
-            }
-          },
-          {
-            "model": "llama3.2:3b",
-            "modified_at": "2025-05-01T11:15:23.797447-07:00",
-            "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
-            "size": 2019393189,
-            "details": {
-              "parent_model": "",
-              "format": "gguf",
-              "family": "llama",
-              "families": [
-                "llama"
-              ],
-              "parameter_size": "3.2B",
-              "quantization_level": "Q4_K_M"
-            }
-          },
           {
             "model": "llama3.2:3b-instruct-fp16",
-            "modified_at": "2025-04-30T15:33:48.939665-07:00",
+            "modified_at": "2025-07-31T04:42:05Z",
             "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d",
             "size": 6433703586,
             "details": {
diff --git a/tests/integration/recordings/responses/561746e1c8de.json b/tests/integration/recordings/responses/561746e1c8de.json
index a28366693..120f40661 100644
--- a/tests/integration/recordings/responses/561746e1c8de.json
+++ b/tests/integration/recordings/responses/561746e1c8de.json
@@ -21,7 +21,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.141947Z",
+          "created_at": "2025-08-15T20:24:49.18651486Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -39,7 +39,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.194979Z",
+          "created_at": "2025-08-15T20:24:49.370611348Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -57,7 +57,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.248312Z",
+          "created_at": "2025-08-15T20:24:49.557000029Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -75,7 +75,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.301911Z",
+          "created_at": "2025-08-15T20:24:49.746777116Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -93,7 +93,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.354437Z",
+          "created_at": "2025-08-15T20:24:49.942233333Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -111,7 +111,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.406821Z",
+          "created_at": "2025-08-15T20:24:50.126788846Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -129,7 +129,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.457633Z",
+          "created_at": "2025-08-15T20:24:50.311346131Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -147,7 +147,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.507857Z",
+          "created_at": "2025-08-15T20:24:50.501507173Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -165,7 +165,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.558847Z",
+          "created_at": "2025-08-15T20:24:50.692296777Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -183,7 +183,7 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.609969Z",
+          "created_at": "2025-08-15T20:24:50.878846539Z",
           "done": false,
           "done_reason": null,
           "total_duration": null,
@@ -201,15 +201,15 @@
         "__type__": "ollama._types.GenerateResponse",
         "__data__": {
           "model": "llama3.2:3b-instruct-fp16",
-          "created_at": "2025-08-04T22:55:14.660997Z",
+          "created_at": "2025-08-15T20:24:51.063200561Z",
           "done": true,
           "done_reason": "stop",
-          "total_duration": 715356542,
-          "load_duration": 59747500,
+          "total_duration": 33982453650,
+          "load_duration": 2909001805,
           "prompt_eval_count": 341,
-          "prompt_eval_duration": 128000000,
+          "prompt_eval_duration": 29194357307,
           "eval_count": 11,
-          "eval_duration": 526000000,
+          "eval_duration": 1878247732,
           "response": "",
           "thinking": null,
           "context": null

From f4ccdee20083c2dc2fa18d8a243b647927e26e35 Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 15:30:03 -0700
Subject: [PATCH 43/45] fix(ci): skip batches directory for library client
 testing

---
 scripts/integration-tests.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh
index 95b78e271..bf7671348 100755
--- a/scripts/integration-tests.sh
+++ b/scripts/integration-tests.sh
@@ -221,6 +221,11 @@ for test_subdir in $(echo "$TEST_SUBDIRS" | tr ',' '\n'); do
         fi
     fi
 
+    if [[ "$STACK_CONFIG" != *"server:"* ]] && [[ "$test_subdir" == "batches" ]]; then
+        echo "Skipping $test_subdir for library client until types are supported"
+        continue
+    fi
+
     if [[ -d "tests/integration/$test_subdir" ]]; then
         # Find all Python test files in this directory
         test_files=$(find tests/integration/$test_subdir -name "test_*.py" -o -name "*_test.py")

From 914c7be288440169a9775ed4bda2fef43116d468 Mon Sep 17 00:00:00 2001
From: Matthew Farrellee <matt@cs.wisc.edu>
Date: Fri, 15 Aug 2025 17:34:15 -0500
Subject: [PATCH 44/45] feat: add batches API with OpenAI compatibility (with
 inference replay) (#3162)

Add complete batches API implementation with protocol, providers, and
tests:

Core Infrastructure:
- Add batches API protocol using OpenAI Batch types directly
- Add Api.batches enum value and protocol mapping in resolver
- Add OpenAI "batch" file purpose support
- Include proper error handling (ConflictError, ResourceNotFoundError)

Reference Provider:
- Add ReferenceBatchesImpl with full CRUD operations (create, retrieve,
cancel, list)
- Implement background batch processing with configurable concurrency
- Add SQLite KVStore backend for persistence
- Support /v1/chat/completions endpoint with request validation

Comprehensive Test Suite:
- Add unit tests for provider implementation with validation
- Add integration tests for end-to-end batch processing workflows
- Add error handling tests for validation, malformed inputs, and edge
cases

Configuration:
- Add max_concurrent_batches and max_concurrent_requests_per_batch
options
- Add provider documentation with sample configurations

Test with -

```
$ uv run llama stack build --image-type venv --providers inference=YOU_PICK,files=inline::localfs,batches=inline::reference --run &
$ LLAMA_STACK_CONFIG=http://localhost:8321 uv run pytest tests/unit/providers/batches tests/integration/batches --text-model YOU_PICK
```

addresses #3066

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Ashwin Bharambe <ashwin.bharambe@gmail.com>
---
 docs/_static/llama-stack-spec.html            |   6 +-
 docs/_static/llama-stack-spec.yaml            |   2 +
 docs/source/concepts/apis.md                  |   1 +
 docs/source/providers/agents/index.md         |   9 +
 docs/source/providers/batches/index.md        |  21 +
 .../providers/batches/inline_reference.md     |  23 +
 docs/source/providers/eval/index.md           |   2 +
 docs/source/providers/inference/index.md      |   6 +
 llama_stack/apis/batches/__init__.py          |   9 +
 llama_stack/apis/batches/batches.py           |  89 +++
 llama_stack/apis/common/errors.py             |   7 +
 llama_stack/apis/datatypes.py                 |   2 +
 llama_stack/apis/files/files.py               |   1 +
 llama_stack/core/resolver.py                  |   2 +
 llama_stack/core/server/server.py             |   5 +
 llama_stack/distributions/ci-tests/build.yaml |   2 +
 llama_stack/distributions/ci-tests/run.yaml   |   8 +
 llama_stack/distributions/starter/build.yaml  |   2 +
 llama_stack/distributions/starter/run.yaml    |   8 +
 llama_stack/distributions/starter/starter.py  |   3 +
 .../providers/inline/batches/__init__.py      |   5 +
 .../inline/batches/reference/__init__.py      |  36 +
 .../inline/batches/reference/batches.py       | 580 ++++++++++++++
 .../inline/batches/reference/config.py        |  40 +
 llama_stack/providers/registry/batches.py     |  26 +
 scripts/provider_codegen.py                   |  22 +
 tests/integration/batches/__init__.py         |   5 +
 tests/integration/batches/conftest.py         | 122 +++
 tests/integration/batches/test_batches.py     | 270 +++++++
 .../batches/test_batches_errors.py            | 693 ++++++++++++++++
 tests/integration/recordings/index.sqlite     | Bin 53248 -> 57344 bytes
 .../recordings/responses/3c0bf9ba81b2.json    |  56 ++
 .../recordings/responses/44a1d9de0602.json    |  56 ++
 .../recordings/responses/4a3a4447b16b.json    |   2 +-
 .../unit/providers/batches/test_reference.py  | 753 ++++++++++++++++++
 35 files changed, 2871 insertions(+), 3 deletions(-)
 create mode 100644 docs/source/providers/batches/index.md
 create mode 100644 docs/source/providers/batches/inline_reference.md
 create mode 100644 llama_stack/apis/batches/__init__.py
 create mode 100644 llama_stack/apis/batches/batches.py
 create mode 100644 llama_stack/providers/inline/batches/__init__.py
 create mode 100644 llama_stack/providers/inline/batches/reference/__init__.py
 create mode 100644 llama_stack/providers/inline/batches/reference/batches.py
 create mode 100644 llama_stack/providers/inline/batches/reference/config.py
 create mode 100644 llama_stack/providers/registry/batches.py
 create mode 100644 tests/integration/batches/__init__.py
 create mode 100644 tests/integration/batches/conftest.py
 create mode 100644 tests/integration/batches/test_batches.py
 create mode 100644 tests/integration/batches/test_batches_errors.py
 create mode 100644 tests/integration/recordings/responses/3c0bf9ba81b2.json
 create mode 100644 tests/integration/recordings/responses/44a1d9de0602.json
 create mode 100644 tests/unit/providers/batches/test_reference.py

diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index 0549dda21..b36626719 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -14767,7 +14767,8 @@
             "OpenAIFilePurpose": {
                 "type": "string",
                 "enum": [
-                    "assistants"
+                    "assistants",
+                    "batch"
                 ],
                 "title": "OpenAIFilePurpose",
                 "description": "Valid purpose values for OpenAI Files API."
@@ -14844,7 +14845,8 @@
                     "purpose": {
                         "type": "string",
                         "enum": [
-                            "assistants"
+                            "assistants",
+                            "batch"
                         ],
                         "description": "The intended purpose of the file"
                     }
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index aa47cd58d..e7733b3c3 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -10951,6 +10951,7 @@ components:
       type: string
       enum:
         - assistants
+        - batch
       title: OpenAIFilePurpose
       description: >-
         Valid purpose values for OpenAI Files API.
@@ -11019,6 +11020,7 @@ components:
           type: string
           enum:
             - assistants
+            - batch
           description: The intended purpose of the file
       additionalProperties: false
       required:
diff --git a/docs/source/concepts/apis.md b/docs/source/concepts/apis.md
index 5a10d6498..f8f73a928 100644
--- a/docs/source/concepts/apis.md
+++ b/docs/source/concepts/apis.md
@@ -18,3 +18,4 @@ We are working on adding a few more APIs to complete the application lifecycle.
 - **Batch Inference**: run inference on a dataset of inputs
 - **Batch Agents**: run agents on a dataset of inputs
 - **Synthetic Data Generation**: generate synthetic data for model development
+- **Batches**: OpenAI-compatible batch management for inference
diff --git a/docs/source/providers/agents/index.md b/docs/source/providers/agents/index.md
index 92bf9edc0..a2c48d4b9 100644
--- a/docs/source/providers/agents/index.md
+++ b/docs/source/providers/agents/index.md
@@ -2,6 +2,15 @@
 
 ## Overview
 
+Agents API for creating and interacting with agentic systems.
+
+    Main functionalities provided by this API:
+    - Create agents with specific instructions and ability to use tools.
+    - Interactions with agents are grouped into sessions ("threads"), and each interaction is called a "turn".
+    - Agents can be provided with various tools (see the ToolGroups and ToolRuntime APIs for more details).
+    - Agents can be provided with various shields (see the Safety API for more details).
+    - Agents can also use Memory to retrieve information from knowledge bases. See the RAG Tool and Vector IO APIs for more details.
+
 This section contains documentation for all available providers for the **agents** API.
 
 ## Providers
diff --git a/docs/source/providers/batches/index.md b/docs/source/providers/batches/index.md
new file mode 100644
index 000000000..2a39a626c
--- /dev/null
+++ b/docs/source/providers/batches/index.md
@@ -0,0 +1,21 @@
+# Batches
+
+## Overview
+
+Protocol for batch processing API operations.
+
+    The Batches API enables efficient processing of multiple requests in a single operation,
+    particularly useful for processing large datasets, batch evaluation workflows, and
+    cost-effective inference at scale.
+
+    Note: This API is currently under active development and may undergo changes.
+
+This section contains documentation for all available providers for the **batches** API.
+
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_reference
+```
diff --git a/docs/source/providers/batches/inline_reference.md b/docs/source/providers/batches/inline_reference.md
new file mode 100644
index 000000000..a58e5124d
--- /dev/null
+++ b/docs/source/providers/batches/inline_reference.md
@@ -0,0 +1,23 @@
+# inline::reference
+
+## Description
+
+Reference implementation of batches API with KVStore persistence.
+
+## Configuration
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Configuration for the key-value store backend. |
+| `max_concurrent_batches` | `<class 'int'>` | No | 1 | Maximum number of concurrent batches to process simultaneously. |
+| `max_concurrent_requests_per_batch` | `<class 'int'>` | No | 10 | Maximum number of concurrent requests to process per batch. |
+
+## Sample Configuration
+
+```yaml
+kvstore:
+  type: sqlite
+  db_path: ${env.SQLITE_STORE_DIR:=~/.llama/dummy}/batches.db
+
+```
+
diff --git a/docs/source/providers/eval/index.md b/docs/source/providers/eval/index.md
index d180d256c..a14fada1d 100644
--- a/docs/source/providers/eval/index.md
+++ b/docs/source/providers/eval/index.md
@@ -2,6 +2,8 @@
 
 ## Overview
 
+Llama Stack Evaluation API for running evaluations on model and agent candidates.
+
 This section contains documentation for all available providers for the **eval** API.
 
 ## Providers
diff --git a/docs/source/providers/inference/index.md b/docs/source/providers/inference/index.md
index 38781e5eb..b6d215474 100644
--- a/docs/source/providers/inference/index.md
+++ b/docs/source/providers/inference/index.md
@@ -2,6 +2,12 @@
 
 ## Overview
 
+Llama Stack Inference API for generating completions, chat completions, and embeddings.
+
+    This API provides the raw interface to the underlying models. Two kinds of models are supported:
+    - LLM models: these models generate "raw" and "chat" (conversational) completions.
+    - Embedding models: these models generate embeddings to be used for semantic search.
+
 This section contains documentation for all available providers for the **inference** API.
 
 ## Providers
diff --git a/llama_stack/apis/batches/__init__.py b/llama_stack/apis/batches/__init__.py
new file mode 100644
index 000000000..9ce7d3d75
--- /dev/null
+++ b/llama_stack/apis/batches/__init__.py
@@ -0,0 +1,9 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from .batches import Batches, BatchObject, ListBatchesResponse
+
+__all__ = ["Batches", "BatchObject", "ListBatchesResponse"]
diff --git a/llama_stack/apis/batches/batches.py b/llama_stack/apis/batches/batches.py
new file mode 100644
index 000000000..9297d8597
--- /dev/null
+++ b/llama_stack/apis/batches/batches.py
@@ -0,0 +1,89 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Literal, Protocol, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+from llama_stack.schema_utils import json_schema_type, webmethod
+
+try:
+    from openai.types import Batch as BatchObject
+except ImportError as e:
+    raise ImportError("OpenAI package is required for batches API. Please install it with: pip install openai") from e
+
+
+@json_schema_type
+class ListBatchesResponse(BaseModel):
+    """Response containing a list of batch objects."""
+
+    object: Literal["list"] = "list"
+    data: list[BatchObject] = Field(..., description="List of batch objects")
+    first_id: str | None = Field(default=None, description="ID of the first batch in the list")
+    last_id: str | None = Field(default=None, description="ID of the last batch in the list")
+    has_more: bool = Field(default=False, description="Whether there are more batches available")
+
+
+@runtime_checkable
+class Batches(Protocol):
+    """Protocol for batch processing API operations.
+
+    The Batches API enables efficient processing of multiple requests in a single operation,
+    particularly useful for processing large datasets, batch evaluation workflows, and
+    cost-effective inference at scale.
+
+    Note: This API is currently under active development and may undergo changes.
+    """
+
+    @webmethod(route="/openai/v1/batches", method="POST")
+    async def create_batch(
+        self,
+        input_file_id: str,
+        endpoint: str,
+        completion_window: Literal["24h"],
+        metadata: dict[str, str] | None = None,
+    ) -> BatchObject:
+        """Create a new batch for processing multiple API requests.
+
+        :param input_file_id: The ID of an uploaded file containing requests for the batch.
+        :param endpoint: The endpoint to be used for all requests in the batch.
+        :param completion_window: The time window within which the batch should be processed.
+        :param metadata: Optional metadata for the batch.
+        :returns: The created batch object.
+        """
+        ...
+
+    @webmethod(route="/openai/v1/batches/{batch_id}", method="GET")
+    async def retrieve_batch(self, batch_id: str) -> BatchObject:
+        """Retrieve information about a specific batch.
+
+        :param batch_id: The ID of the batch to retrieve.
+        :returns: The batch object.
+        """
+        ...
+
+    @webmethod(route="/openai/v1/batches/{batch_id}/cancel", method="POST")
+    async def cancel_batch(self, batch_id: str) -> BatchObject:
+        """Cancel a batch that is in progress.
+
+        :param batch_id: The ID of the batch to cancel.
+        :returns: The updated batch object.
+        """
+        ...
+
+    @webmethod(route="/openai/v1/batches", method="GET")
+    async def list_batches(
+        self,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> ListBatchesResponse:
+        """List all batches for the current user.
+
+        :param after: A cursor for pagination; returns batches after this batch ID.
+        :param limit: Number of batches to return (default 20, max 100).
+        :returns: A list of batch objects.
+        """
+        ...
diff --git a/llama_stack/apis/common/errors.py b/llama_stack/apis/common/errors.py
index 6e0fa0b3c..ec3d2b1ce 100644
--- a/llama_stack/apis/common/errors.py
+++ b/llama_stack/apis/common/errors.py
@@ -72,3 +72,10 @@ class ModelTypeError(TypeError):
             f"Model '{model_name}' is of type '{model_type}' rather than the expected type '{expected_model_type}'"
         )
         super().__init__(message)
+
+
+class ConflictError(ValueError):
+    """raised when an operation cannot be performed due to a conflict with the current state"""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
diff --git a/llama_stack/apis/datatypes.py b/llama_stack/apis/datatypes.py
index cabe46a2f..87fc95917 100644
--- a/llama_stack/apis/datatypes.py
+++ b/llama_stack/apis/datatypes.py
@@ -86,6 +86,7 @@ class Api(Enum, metaclass=DynamicApiMeta):
     :cvar inference: Text generation, chat completions, and embeddings
     :cvar safety: Content moderation and safety shields
     :cvar agents: Agent orchestration and execution
+    :cvar batches: Batch processing for asynchronous API requests
     :cvar vector_io: Vector database operations and queries
     :cvar datasetio: Dataset input/output operations
     :cvar scoring: Model output evaluation and scoring
@@ -108,6 +109,7 @@ class Api(Enum, metaclass=DynamicApiMeta):
     inference = "inference"
     safety = "safety"
     agents = "agents"
+    batches = "batches"
     vector_io = "vector_io"
     datasetio = "datasetio"
     scoring = "scoring"
diff --git a/llama_stack/apis/files/files.py b/llama_stack/apis/files/files.py
index ba8701e23..a1b9dd4dc 100644
--- a/llama_stack/apis/files/files.py
+++ b/llama_stack/apis/files/files.py
@@ -22,6 +22,7 @@ class OpenAIFilePurpose(StrEnum):
     """
 
     ASSISTANTS = "assistants"
+    BATCH = "batch"
     # TODO: Add other purposes as needed
 
 
diff --git a/llama_stack/core/resolver.py b/llama_stack/core/resolver.py
index 70c78fb01..7ac98dac8 100644
--- a/llama_stack/core/resolver.py
+++ b/llama_stack/core/resolver.py
@@ -8,6 +8,7 @@ import inspect
 from typing import Any
 
 from llama_stack.apis.agents import Agents
+from llama_stack.apis.batches import Batches
 from llama_stack.apis.benchmarks import Benchmarks
 from llama_stack.apis.datasetio import DatasetIO
 from llama_stack.apis.datasets import Datasets
@@ -75,6 +76,7 @@ def api_protocol_map(external_apis: dict[Api, ExternalApiSpec] | None = None) ->
         Api.agents: Agents,
         Api.inference: Inference,
         Api.inspect: Inspect,
+        Api.batches: Batches,
         Api.vector_io: VectorIO,
         Api.vector_dbs: VectorDBs,
         Api.models: Models,
diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py
index e9d70fc8d..cbef8ef88 100644
--- a/llama_stack/core/server/server.py
+++ b/llama_stack/core/server/server.py
@@ -32,6 +32,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
 from openai import BadRequestError
 from pydantic import BaseModel, ValidationError
 
+from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
 from llama_stack.apis.common.responses import PaginatedResponse
 from llama_stack.cli.utils import add_config_distro_args, get_config_from_args
 from llama_stack.core.access_control.access_control import AccessDeniedError
@@ -128,6 +129,10 @@ def translate_exception(exc: Exception) -> HTTPException | RequestValidationErro
                 ]
             },
         )
+    elif isinstance(exc, ConflictError):
+        return HTTPException(status_code=409, detail=str(exc))
+    elif isinstance(exc, ResourceNotFoundError):
+        return HTTPException(status_code=404, detail=str(exc))
     elif isinstance(exc, ValueError):
         return HTTPException(status_code=httpx.codes.BAD_REQUEST, detail=f"Invalid value: {str(exc)}")
     elif isinstance(exc, BadRequestError):
diff --git a/llama_stack/distributions/ci-tests/build.yaml b/llama_stack/distributions/ci-tests/build.yaml
index e6e699b62..676ed18d2 100644
--- a/llama_stack/distributions/ci-tests/build.yaml
+++ b/llama_stack/distributions/ci-tests/build.yaml
@@ -48,6 +48,8 @@ distribution_spec:
     - provider_type: remote::tavily-search
     - provider_type: inline::rag-runtime
     - provider_type: remote::model-context-protocol
+    batches:
+    - provider_type: inline::reference
 image_type: venv
 additional_pip_packages:
 - aiosqlite
diff --git a/llama_stack/distributions/ci-tests/run.yaml b/llama_stack/distributions/ci-tests/run.yaml
index 05e1b4576..dd4e04e50 100644
--- a/llama_stack/distributions/ci-tests/run.yaml
+++ b/llama_stack/distributions/ci-tests/run.yaml
@@ -2,6 +2,7 @@ version: 2
 image_name: ci-tests
 apis:
 - agents
+- batches
 - datasetio
 - eval
 - files
@@ -204,6 +205,13 @@ providers:
     provider_type: inline::rag-runtime
   - provider_id: model-context-protocol
     provider_type: remote::model-context-protocol
+  batches:
+  - provider_id: reference
+    provider_type: inline::reference
+    config:
+      kvstore:
+        type: sqlite
+        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/batches.db
 metadata_store:
   type: sqlite
   db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/registry.db
diff --git a/llama_stack/distributions/starter/build.yaml b/llama_stack/distributions/starter/build.yaml
index 1a4f81d49..549bb4529 100644
--- a/llama_stack/distributions/starter/build.yaml
+++ b/llama_stack/distributions/starter/build.yaml
@@ -48,6 +48,8 @@ distribution_spec:
     - provider_type: remote::tavily-search
     - provider_type: inline::rag-runtime
     - provider_type: remote::model-context-protocol
+    batches:
+    - provider_type: inline::reference
 image_type: venv
 additional_pip_packages:
 - aiosqlite
diff --git a/llama_stack/distributions/starter/run.yaml b/llama_stack/distributions/starter/run.yaml
index 46bd12956..d64c275cb 100644
--- a/llama_stack/distributions/starter/run.yaml
+++ b/llama_stack/distributions/starter/run.yaml
@@ -2,6 +2,7 @@ version: 2
 image_name: starter
 apis:
 - agents
+- batches
 - datasetio
 - eval
 - files
@@ -204,6 +205,13 @@ providers:
     provider_type: inline::rag-runtime
   - provider_id: model-context-protocol
     provider_type: remote::model-context-protocol
+  batches:
+  - provider_id: reference
+    provider_type: inline::reference
+    config:
+      kvstore:
+        type: sqlite
+        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/batches.db
 metadata_store:
   type: sqlite
   db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/registry.db
diff --git a/llama_stack/distributions/starter/starter.py b/llama_stack/distributions/starter/starter.py
index 0270b68ad..498a12080 100644
--- a/llama_stack/distributions/starter/starter.py
+++ b/llama_stack/distributions/starter/starter.py
@@ -139,6 +139,9 @@ def get_distribution_template() -> DistributionTemplate:
             BuildProvider(provider_type="inline::rag-runtime"),
             BuildProvider(provider_type="remote::model-context-protocol"),
         ],
+        "batches": [
+            BuildProvider(provider_type="inline::reference"),
+        ],
     }
     files_provider = Provider(
         provider_id="meta-reference-files",
diff --git a/llama_stack/providers/inline/batches/__init__.py b/llama_stack/providers/inline/batches/__init__.py
new file mode 100644
index 000000000..756f351d8
--- /dev/null
+++ b/llama_stack/providers/inline/batches/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
diff --git a/llama_stack/providers/inline/batches/reference/__init__.py b/llama_stack/providers/inline/batches/reference/__init__.py
new file mode 100644
index 000000000..a8ae92eb2
--- /dev/null
+++ b/llama_stack/providers/inline/batches/reference/__init__.py
@@ -0,0 +1,36 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any
+
+from llama_stack.apis.files import Files
+from llama_stack.apis.inference import Inference
+from llama_stack.apis.models import Models
+from llama_stack.core.datatypes import AccessRule, Api
+from llama_stack.providers.utils.kvstore import kvstore_impl
+
+from .batches import ReferenceBatchesImpl
+from .config import ReferenceBatchesImplConfig
+
+__all__ = ["ReferenceBatchesImpl", "ReferenceBatchesImplConfig"]
+
+
+async def get_provider_impl(config: ReferenceBatchesImplConfig, deps: dict[Api, Any], policy: list[AccessRule]):
+    kvstore = await kvstore_impl(config.kvstore)
+    inference_api: Inference | None = deps.get(Api.inference)
+    files_api: Files | None = deps.get(Api.files)
+    models_api: Models | None = deps.get(Api.models)
+
+    if inference_api is None:
+        raise ValueError("Inference API is required but not provided in dependencies")
+    if files_api is None:
+        raise ValueError("Files API is required but not provided in dependencies")
+    if models_api is None:
+        raise ValueError("Models API is required but not provided in dependencies")
+
+    impl = ReferenceBatchesImpl(config, inference_api, files_api, models_api, kvstore)
+    await impl.initialize()
+    return impl
diff --git a/llama_stack/providers/inline/batches/reference/batches.py b/llama_stack/providers/inline/batches/reference/batches.py
new file mode 100644
index 000000000..1ff554e70
--- /dev/null
+++ b/llama_stack/providers/inline/batches/reference/batches.py
@@ -0,0 +1,580 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import asyncio
+import itertools
+import json
+import time
+import uuid
+from io import BytesIO
+from typing import Any, Literal
+
+from openai.types.batch import BatchError, Errors
+from pydantic import BaseModel
+
+from llama_stack.apis.batches import Batches, BatchObject, ListBatchesResponse
+from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
+from llama_stack.apis.files import Files, OpenAIFilePurpose
+from llama_stack.apis.inference import (
+    Inference,
+    OpenAIAssistantMessageParam,
+    OpenAIDeveloperMessageParam,
+    OpenAIMessageParam,
+    OpenAISystemMessageParam,
+    OpenAIToolMessageParam,
+    OpenAIUserMessageParam,
+)
+from llama_stack.apis.models import Models
+from llama_stack.log import get_logger
+from llama_stack.providers.utils.kvstore import KVStore
+
+from .config import ReferenceBatchesImplConfig
+
+BATCH_PREFIX = "batch:"
+
+logger = get_logger(__name__)
+
+
+class AsyncBytesIO:
+    """
+    Async-compatible BytesIO wrapper to allow async file-like operations.
+
+    We use this when uploading files to the Files API, as it expects an
+    async file-like object.
+    """
+
+    def __init__(self, data: bytes):
+        self._buffer = BytesIO(data)
+
+    async def read(self, n=-1):
+        return self._buffer.read(n)
+
+    async def seek(self, pos, whence=0):
+        return self._buffer.seek(pos, whence)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._buffer.close()
+
+    def __getattr__(self, name):
+        return getattr(self._buffer, name)
+
+
+class BatchRequest(BaseModel):
+    line_num: int
+    custom_id: str
+    method: str
+    url: str
+    body: dict[str, Any]
+
+
+def convert_to_openai_message_param(msg: dict[str, Any]) -> OpenAIMessageParam:
+    """Convert a message dictionary to OpenAIMessageParam based on role."""
+    role = msg.get("role")
+
+    if role == "user":
+        return OpenAIUserMessageParam(**msg)
+    elif role == "system":
+        return OpenAISystemMessageParam(**msg)
+    elif role == "assistant":
+        return OpenAIAssistantMessageParam(**msg)
+    elif role == "tool":
+        return OpenAIToolMessageParam(**msg)
+    elif role == "developer":
+        return OpenAIDeveloperMessageParam(**msg)
+    else:
+        raise ValueError(f"Unknown message role: {role}")
+
+
+class ReferenceBatchesImpl(Batches):
+    """Reference implementation of the Batches API.
+
+    This implementation processes batch files by making individual requests
+    to the inference API and generates output files with results.
+    """
+
+    def __init__(
+        self,
+        config: ReferenceBatchesImplConfig,
+        inference_api: Inference,
+        files_api: Files,
+        models_api: Models,
+        kvstore: KVStore,
+    ) -> None:
+        self.config = config
+        self.kvstore = kvstore
+        self.inference_api = inference_api
+        self.files_api = files_api
+        self.models_api = models_api
+        self._processing_tasks: dict[str, asyncio.Task] = {}
+        self._batch_semaphore = asyncio.Semaphore(config.max_concurrent_batches)
+        self._update_batch_lock = asyncio.Lock()
+
+        # this is to allow tests to disable background processing
+        self.process_batches = True
+
+    async def initialize(self) -> None:
+        # TODO: start background processing of existing tasks
+        pass
+
+    async def shutdown(self) -> None:
+        """Shutdown the batches provider."""
+        if self._processing_tasks:
+            # don't cancel tasks - just let them stop naturally on shutdown
+            # cancelling would mark batches as "cancelled" in the database
+            logger.info(f"Shutdown initiated with {len(self._processing_tasks)} active batch processing tasks")
+
+    # TODO (SECURITY): this currently works w/ configured api keys, not with x-llamastack-provider-data or with user policy restrictions
+    async def create_batch(
+        self,
+        input_file_id: str,
+        endpoint: str,
+        completion_window: Literal["24h"],
+        metadata: dict[str, str] | None = None,
+    ) -> BatchObject:
+        """
+        Create a new batch for processing multiple API requests.
+
+        Error handling by levels -
+         0. Input param handling, results in 40x errors before processing, e.g.
+           - Wrong completion_window
+           - Invalid metadata types
+           - Unknown endpoint
+          -> no batch created
+         1. Errors preventing processing, result in BatchErrors aggregated in process_batch, e.g.
+           - input_file_id missing
+           - invalid json in file
+           - missing custom_id, method, url, body
+           - invalid model
+           - streaming
+          -> batch created, validation sends to failed status
+         2. Processing errors, result in error_file_id entries, e.g.
+           - Any error returned from inference endpoint
+          -> batch created, goes to completed status
+        """
+
+        # TODO: set expiration time for garbage collection
+
+        if endpoint not in ["/v1/chat/completions"]:
+            raise ValueError(
+                f"Invalid endpoint: {endpoint}. Supported values: /v1/chat/completions. Code: invalid_value. Param: endpoint",
+            )
+
+        if completion_window != "24h":
+            raise ValueError(
+                f"Invalid completion_window: {completion_window}. Supported values are: 24h. Code: invalid_value. Param: completion_window",
+            )
+
+        batch_id = f"batch_{uuid.uuid4().hex[:16]}"
+        current_time = int(time.time())
+
+        batch = BatchObject(
+            id=batch_id,
+            object="batch",
+            endpoint=endpoint,
+            input_file_id=input_file_id,
+            completion_window=completion_window,
+            status="validating",
+            created_at=current_time,
+            metadata=metadata,
+        )
+
+        await self.kvstore.set(f"batch:{batch_id}", batch.to_json())
+
+        if self.process_batches:
+            task = asyncio.create_task(self._process_batch(batch_id))
+            self._processing_tasks[batch_id] = task
+
+        return batch
+
+    async def cancel_batch(self, batch_id: str) -> BatchObject:
+        """Cancel a batch that is in progress."""
+        batch = await self.retrieve_batch(batch_id)
+
+        if batch.status in ["cancelled", "cancelling"]:
+            return batch
+
+        if batch.status in ["completed", "failed", "expired"]:
+            raise ConflictError(f"Cannot cancel batch '{batch_id}' with status '{batch.status}'")
+
+        await self._update_batch(batch_id, status="cancelling", cancelling_at=int(time.time()))
+
+        if batch_id in self._processing_tasks:
+            self._processing_tasks[batch_id].cancel()
+            # note: task removal and status="cancelled" handled in finally block of _process_batch
+
+        return await self.retrieve_batch(batch_id)
+
+    async def list_batches(
+        self,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> ListBatchesResponse:
+        """
+        List all batches, eventually only for the current user.
+
+        With no notion of user, we return all batches.
+        """
+        batch_values = await self.kvstore.values_in_range("batch:", "batch:\xff")
+
+        batches = []
+        for batch_data in batch_values:
+            if batch_data:
+                batches.append(BatchObject.model_validate_json(batch_data))
+
+        batches.sort(key=lambda b: b.created_at, reverse=True)
+
+        start_idx = 0
+        if after:
+            for i, batch in enumerate(batches):
+                if batch.id == after:
+                    start_idx = i + 1
+                    break
+
+        page_batches = batches[start_idx : start_idx + limit]
+        has_more = (start_idx + limit) < len(batches)
+
+        first_id = page_batches[0].id if page_batches else None
+        last_id = page_batches[-1].id if page_batches else None
+
+        return ListBatchesResponse(
+            data=page_batches,
+            first_id=first_id,
+            last_id=last_id,
+            has_more=has_more,
+        )
+
+    async def retrieve_batch(self, batch_id: str) -> BatchObject:
+        """Retrieve information about a specific batch."""
+        batch_data = await self.kvstore.get(f"batch:{batch_id}")
+        if not batch_data:
+            raise ResourceNotFoundError(batch_id, "Batch", "batches.list()")
+
+        return BatchObject.model_validate_json(batch_data)
+
+    async def _update_batch(self, batch_id: str, **updates) -> None:
+        """Update batch fields in kvstore."""
+        async with self._update_batch_lock:
+            try:
+                batch = await self.retrieve_batch(batch_id)
+
+                # batch processing is async. once cancelling, only allow "cancelled" status updates
+                if batch.status == "cancelling" and updates.get("status") != "cancelled":
+                    logger.info(
+                        f"Skipping status update for cancelled batch {batch_id}: attempted {updates.get('status')}"
+                    )
+                    return
+
+                if "errors" in updates:
+                    updates["errors"] = updates["errors"].model_dump()
+
+                batch_dict = batch.model_dump()
+                batch_dict.update(updates)
+
+                await self.kvstore.set(f"batch:{batch_id}", json.dumps(batch_dict))
+            except Exception as e:
+                logger.error(f"Failed to update batch {batch_id}: {e}")
+
+    async def _validate_input(self, batch: BatchObject) -> tuple[list[BatchError], list[BatchRequest]]:
+        """
+        Read & validate input, return errors and valid input.
+
+        Validation of
+        - input_file_id existance
+        - valid json
+        - custom_id, method, url, body presence and valid
+        - no streaming
+        """
+        requests: list[BatchRequest] = []
+        errors: list[BatchError] = []
+        try:
+            await self.files_api.openai_retrieve_file(batch.input_file_id)
+        except Exception:
+            errors.append(
+                BatchError(
+                    code="invalid_request",
+                    line=None,
+                    message=f"Cannot find file {batch.input_file_id}.",
+                    param="input_file_id",
+                )
+            )
+            return errors, requests
+
+        # TODO(SECURITY): do something about large files
+        file_content_response = await self.files_api.openai_retrieve_file_content(batch.input_file_id)
+        file_content = file_content_response.body.decode("utf-8")
+        for line_num, line in enumerate(file_content.strip().split("\n"), 1):
+            if line.strip():  # skip empty lines
+                try:
+                    request = json.loads(line)
+
+                    if not isinstance(request, dict):
+                        errors.append(
+                            BatchError(
+                                code="invalid_request",
+                                line=line_num,
+                                message="Each line must be a JSON dictionary object",
+                            )
+                        )
+                        continue
+
+                    valid = True
+
+                    for param, expected_type, type_string in [
+                        ("custom_id", str, "string"),
+                        ("method", str, "string"),
+                        ("url", str, "string"),
+                        ("body", dict, "JSON dictionary object"),
+                    ]:
+                        if param not in request:
+                            errors.append(
+                                BatchError(
+                                    code="missing_required_parameter",
+                                    line=line_num,
+                                    message=f"Missing required parameter: {param}",
+                                    param=param,
+                                )
+                            )
+                            valid = False
+                        elif not isinstance(request[param], expected_type):
+                            param_name = "URL" if param == "url" else param.capitalize()
+                            errors.append(
+                                BatchError(
+                                    code="invalid_request",
+                                    line=line_num,
+                                    message=f"{param_name} must be a {type_string}",
+                                    param=param,
+                                )
+                            )
+                            valid = False
+
+                    if (url := request.get("url")) and isinstance(url, str) and url != batch.endpoint:
+                        errors.append(
+                            BatchError(
+                                code="invalid_url",
+                                line=line_num,
+                                message="URL provided for this request does not match the batch endpoint",
+                                param="url",
+                            )
+                        )
+                        valid = False
+
+                    if (body := request.get("body")) and isinstance(body, dict):
+                        if body.get("stream", False):
+                            errors.append(
+                                BatchError(
+                                    code="streaming_unsupported",
+                                    line=line_num,
+                                    message="Streaming is not supported in batch processing",
+                                    param="body.stream",
+                                )
+                            )
+                            valid = False
+
+                        for param, expected_type, type_string in [
+                            ("model", str, "a string"),
+                            # messages is specific to /v1/chat/completions
+                            # we could skip validating messages here and let inference fail. however,
+                            # that would be a very expensive way to find out messages is wrong.
+                            ("messages", list, "an array"),  # TODO: allow messages to be a string?
+                        ]:
+                            if param not in body:
+                                errors.append(
+                                    BatchError(
+                                        code="invalid_request",
+                                        line=line_num,
+                                        message=f"{param.capitalize()} parameter is required",
+                                        param=f"body.{param}",
+                                    )
+                                )
+                                valid = False
+                            elif not isinstance(body[param], expected_type):
+                                errors.append(
+                                    BatchError(
+                                        code="invalid_request",
+                                        line=line_num,
+                                        message=f"{param.capitalize()} must be {type_string}",
+                                        param=f"body.{param}",
+                                    )
+                                )
+                                valid = False
+
+                        if "model" in body and isinstance(body["model"], str):
+                            try:
+                                await self.models_api.get_model(body["model"])
+                            except Exception:
+                                errors.append(
+                                    BatchError(
+                                        code="model_not_found",
+                                        line=line_num,
+                                        message=f"Model '{body['model']}' does not exist or is not supported",
+                                        param="body.model",
+                                    )
+                                )
+                                valid = False
+
+                    if valid:
+                        assert isinstance(url, str), "URL must be a string"  # for mypy
+                        assert isinstance(body, dict), "Body must be a dictionary"  # for mypy
+                        requests.append(
+                            BatchRequest(
+                                line_num=line_num,
+                                url=url,
+                                method=request["method"],
+                                custom_id=request["custom_id"],
+                                body=body,
+                            ),
+                        )
+                except json.JSONDecodeError:
+                    errors.append(
+                        BatchError(
+                            code="invalid_json_line",
+                            line=line_num,
+                            message="This line is not parseable as valid JSON.",
+                        )
+                    )
+
+        return errors, requests
+
+    async def _process_batch(self, batch_id: str) -> None:
+        """Background task to process a batch of requests."""
+        try:
+            logger.info(f"Starting batch processing for {batch_id}")
+            async with self._batch_semaphore:  # semaphore to limit concurrency
+                logger.info(f"Acquired semaphore for batch {batch_id}")
+                await self._process_batch_impl(batch_id)
+        except asyncio.CancelledError:
+            logger.info(f"Batch processing cancelled for {batch_id}")
+            await self._update_batch(batch_id, status="cancelled", cancelled_at=int(time.time()))
+        except Exception as e:
+            logger.error(f"Batch processing failed for {batch_id}: {e}")
+            await self._update_batch(
+                batch_id,
+                status="failed",
+                failed_at=int(time.time()),
+                errors=Errors(data=[BatchError(code="internal_error", message=str(e))]),
+            )
+        finally:
+            self._processing_tasks.pop(batch_id, None)
+
+    async def _process_batch_impl(self, batch_id: str) -> None:
+        """Implementation of batch processing logic."""
+        errors: list[BatchError] = []
+        batch = await self.retrieve_batch(batch_id)
+
+        errors, requests = await self._validate_input(batch)
+        if errors:
+            await self._update_batch(batch_id, status="failed", failed_at=int(time.time()), errors=Errors(data=errors))
+            logger.info(f"Batch validation failed for {batch_id} with {len(errors)} errors")
+            return
+
+        logger.info(f"Processing {len(requests)} requests for batch {batch_id}")
+
+        total_requests = len(requests)
+        await self._update_batch(
+            batch_id,
+            status="in_progress",
+            request_counts={"total": total_requests, "completed": 0, "failed": 0},
+        )
+
+        error_results = []
+        success_results = []
+        completed_count = 0
+        failed_count = 0
+
+        for chunk in itertools.batched(requests, self.config.max_concurrent_requests_per_batch):
+            # we use a TaskGroup to ensure all process-single-request tasks are canceled when process-batch is cancelled
+            async with asyncio.TaskGroup() as tg:
+                chunk_tasks = [tg.create_task(self._process_single_request(batch_id, request)) for request in chunk]
+
+                chunk_results = await asyncio.gather(*chunk_tasks, return_exceptions=True)
+
+            for result in chunk_results:
+                if isinstance(result, dict) and result.get("error") is not None:  # error response from inference
+                    failed_count += 1
+                    error_results.append(result)
+                elif isinstance(result, dict) and result.get("response") is not None:  # successful inference
+                    completed_count += 1
+                    success_results.append(result)
+                else:  # unexpected result
+                    failed_count += 1
+                    errors.append(BatchError(code="internal_error", message=f"Unexpected result: {result}"))
+
+            await self._update_batch(
+                batch_id,
+                request_counts={"total": total_requests, "completed": completed_count, "failed": failed_count},
+            )
+
+            if errors:
+                await self._update_batch(
+                    batch_id, status="failed", failed_at=int(time.time()), errors=Errors(data=errors)
+                )
+                return
+
+        try:
+            output_file_id = await self._create_output_file(batch_id, success_results, "success")
+            await self._update_batch(batch_id, output_file_id=output_file_id)
+
+            error_file_id = await self._create_output_file(batch_id, error_results, "error")
+            await self._update_batch(batch_id, error_file_id=error_file_id)
+
+            await self._update_batch(batch_id, status="completed", completed_at=int(time.time()))
+
+            logger.info(
+                f"Batch processing completed for {batch_id}: {completed_count} completed, {failed_count} failed"
+            )
+        except Exception as e:
+            # note: errors is empty at this point, so we don't lose anything by ignoring it
+            await self._update_batch(
+                batch_id,
+                status="failed",
+                failed_at=int(time.time()),
+                errors=Errors(data=[BatchError(code="output_failed", message=str(e))]),
+            )
+
+    async def _process_single_request(self, batch_id: str, request: BatchRequest) -> dict:
+        """Process a single request from the batch."""
+        request_id = f"batch_req_{batch_id}_{request.line_num}"
+
+        try:
+            # TODO(SECURITY): review body for security issues
+            request.body["messages"] = [convert_to_openai_message_param(msg) for msg in request.body["messages"]]
+            chat_response = await self.inference_api.openai_chat_completion(**request.body)
+
+            # this is for mypy, we don't allow streaming so we'll get the right type
+            assert hasattr(chat_response, "model_dump_json"), "Chat response must have model_dump_json method"
+            return {
+                "id": request_id,
+                "custom_id": request.custom_id,
+                "response": {
+                    "status_code": 200,
+                    "request_id": request_id,  # TODO: should this be different?
+                    "body": chat_response.model_dump_json(),
+                },
+            }
+        except Exception as e:
+            logger.info(f"Error processing request {request.custom_id} in batch {batch_id}: {e}")
+            return {
+                "id": request_id,
+                "custom_id": request.custom_id,
+                "error": {"type": "request_failed", "message": str(e)},
+            }
+
+    async def _create_output_file(self, batch_id: str, results: list[dict], file_type: str) -> str:
+        """
+        Create an output file with batch results.
+
+        This function filters results based on the specified file_type
+        and uploads the file to the Files API.
+        """
+        output_lines = [json.dumps(result) for result in results]
+
+        with AsyncBytesIO("\n".join(output_lines).encode("utf-8")) as file_buffer:
+            file_buffer.filename = f"{batch_id}_{file_type}.jsonl"
+            uploaded_file = await self.files_api.openai_upload_file(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
+            return uploaded_file.id
diff --git a/llama_stack/providers/inline/batches/reference/config.py b/llama_stack/providers/inline/batches/reference/config.py
new file mode 100644
index 000000000..d8d06868b
--- /dev/null
+++ b/llama_stack/providers/inline/batches/reference/config.py
@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from pydantic import BaseModel, Field
+
+from llama_stack.providers.utils.kvstore.config import KVStoreConfig, SqliteKVStoreConfig
+
+
+class ReferenceBatchesImplConfig(BaseModel):
+    """Configuration for the Reference Batches implementation."""
+
+    kvstore: KVStoreConfig = Field(
+        description="Configuration for the key-value store backend.",
+    )
+
+    max_concurrent_batches: int = Field(
+        default=1,
+        description="Maximum number of concurrent batches to process simultaneously.",
+        ge=1,
+    )
+
+    max_concurrent_requests_per_batch: int = Field(
+        default=10,
+        description="Maximum number of concurrent requests to process per batch.",
+        ge=1,
+    )
+
+    # TODO: add a max requests per second rate limiter
+
+    @classmethod
+    def sample_run_config(cls, __distro_dir__: str) -> dict:
+        return {
+            "kvstore": SqliteKVStoreConfig.sample_run_config(
+                __distro_dir__=__distro_dir__,
+                db_name="batches.db",
+            ),
+        }
diff --git a/llama_stack/providers/registry/batches.py b/llama_stack/providers/registry/batches.py
new file mode 100644
index 000000000..de7886efb
--- /dev/null
+++ b/llama_stack/providers/registry/batches.py
@@ -0,0 +1,26 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+
+from llama_stack.providers.datatypes import Api, InlineProviderSpec, ProviderSpec
+
+
+def available_providers() -> list[ProviderSpec]:
+    return [
+        InlineProviderSpec(
+            api=Api.batches,
+            provider_type="inline::reference",
+            pip_packages=["openai"],
+            module="llama_stack.providers.inline.batches.reference",
+            config_class="llama_stack.providers.inline.batches.reference.config.ReferenceBatchesImplConfig",
+            api_dependencies=[
+                Api.inference,
+                Api.files,
+                Api.models,
+            ],
+            description="Reference implementation of batches API with KVStore persistence.",
+        ),
+    ]
diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py
index 717677c52..060acfa72 100755
--- a/scripts/provider_codegen.py
+++ b/scripts/provider_codegen.py
@@ -18,6 +18,23 @@ from llama_stack.core.distribution import get_provider_registry
 REPO_ROOT = Path(__file__).parent.parent
 
 
+def get_api_docstring(api_name: str) -> str | None:
+    """Extract docstring from the API protocol class."""
+    try:
+        # Import the API module dynamically
+        api_module = __import__(f"llama_stack.apis.{api_name}", fromlist=[api_name.title()])
+
+        # Get the main protocol class (usually capitalized API name)
+        protocol_class_name = api_name.title()
+        if hasattr(api_module, protocol_class_name):
+            protocol_class = getattr(api_module, protocol_class_name)
+            return protocol_class.__doc__
+    except (ImportError, AttributeError):
+        pass
+
+    return None
+
+
 class ChangedPathTracker:
     """Track a list of paths we may have changed."""
 
@@ -261,6 +278,11 @@ def process_provider_registry(progress, change_tracker: ChangedPathTracker) -> N
             index_content.append(f"# {api_name.title()}\n")
             index_content.append("## Overview\n")
 
+            api_docstring = get_api_docstring(api_name)
+            if api_docstring:
+                cleaned_docstring = api_docstring.strip()
+                index_content.append(f"{cleaned_docstring}\n")
+
             index_content.append(
                 f"This section contains documentation for all available providers for the **{api_name}** API.\n"
             )
diff --git a/tests/integration/batches/__init__.py b/tests/integration/batches/__init__.py
new file mode 100644
index 000000000..756f351d8
--- /dev/null
+++ b/tests/integration/batches/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
diff --git a/tests/integration/batches/conftest.py b/tests/integration/batches/conftest.py
new file mode 100644
index 000000000..974fe77ab
--- /dev/null
+++ b/tests/integration/batches/conftest.py
@@ -0,0 +1,122 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Shared pytest fixtures for batch tests."""
+
+import json
+import time
+import warnings
+from contextlib import contextmanager
+from io import BytesIO
+
+import pytest
+
+from llama_stack.apis.files import OpenAIFilePurpose
+
+
+class BatchHelper:
+    """Helper class for creating and managing batch input files."""
+
+    def __init__(self, client):
+        """Initialize with either a batch_client or openai_client."""
+        self.client = client
+
+    @contextmanager
+    def create_file(self, content: str | list[dict], filename_prefix="batch_input"):
+        """Context manager for creating and cleaning up batch input files.
+
+        Args:
+            content: Either a list of batch request dictionaries or raw string content
+            filename_prefix: Prefix for the generated filename (or full filename if content is string)
+
+        Yields:
+            The uploaded file object
+        """
+        if isinstance(content, str):
+            # Handle raw string content (e.g., malformed JSONL, empty files)
+            file_content = content.encode("utf-8")
+        else:
+            # Handle list of batch request dictionaries
+            jsonl_content = "\n".join(json.dumps(req) for req in content)
+            file_content = jsonl_content.encode("utf-8")
+
+        filename = filename_prefix if filename_prefix.endswith(".jsonl") else f"{filename_prefix}.jsonl"
+
+        with BytesIO(file_content) as file_buffer:
+            file_buffer.name = filename
+            uploaded_file = self.client.files.create(file=file_buffer, purpose=OpenAIFilePurpose.BATCH)
+
+        try:
+            yield uploaded_file
+        finally:
+            try:
+                self.client.files.delete(uploaded_file.id)
+            except Exception:
+                warnings.warn(
+                    f"Failed to cleanup file {uploaded_file.id}: {uploaded_file.filename}",
+                    stacklevel=2,
+                )
+
+    def wait_for(
+        self,
+        batch_id: str,
+        max_wait_time: int = 60,
+        sleep_interval: int | None = None,
+        expected_statuses: set[str] | None = None,
+        timeout_action: str = "fail",
+    ):
+        """Wait for a batch to reach a terminal status.
+
+        Args:
+            batch_id: The batch ID to monitor
+            max_wait_time: Maximum time to wait in seconds (default: 60 seconds)
+            sleep_interval: Time to sleep between checks in seconds (default: 1/10th of max_wait_time, min 1s, max 15s)
+            expected_statuses: Set of expected terminal statuses (default: {"completed"})
+            timeout_action: Action on timeout - "fail" (pytest.fail) or "skip" (pytest.skip)
+
+        Returns:
+            The final batch object
+
+        Raises:
+            pytest.Failed: If batch reaches an unexpected status or timeout_action is "fail"
+            pytest.Skipped: If timeout_action is "skip" on timeout or unexpected status
+        """
+        if sleep_interval is None:
+            # Default to 1/10th of max_wait_time, with min 1s and max 15s
+            sleep_interval = max(1, min(15, max_wait_time // 10))
+
+        if expected_statuses is None:
+            expected_statuses = {"completed"}
+
+        terminal_statuses = {"completed", "failed", "cancelled", "expired"}
+        unexpected_statuses = terminal_statuses - expected_statuses
+
+        start_time = time.time()
+        while time.time() - start_time < max_wait_time:
+            current_batch = self.client.batches.retrieve(batch_id)
+
+            if current_batch.status in expected_statuses:
+                return current_batch
+            elif current_batch.status in unexpected_statuses:
+                error_msg = f"Batch reached unexpected status: {current_batch.status}"
+                if timeout_action == "skip":
+                    pytest.skip(error_msg)
+                else:
+                    pytest.fail(error_msg)
+
+            time.sleep(sleep_interval)
+
+        timeout_msg = f"Batch did not reach expected status {expected_statuses} within {max_wait_time} seconds"
+        if timeout_action == "skip":
+            pytest.skip(timeout_msg)
+        else:
+            pytest.fail(timeout_msg)
+
+
+@pytest.fixture
+def batch_helper(openai_client):
+    """Fixture that provides a BatchHelper instance for OpenAI client."""
+    return BatchHelper(openai_client)
diff --git a/tests/integration/batches/test_batches.py b/tests/integration/batches/test_batches.py
new file mode 100644
index 000000000..59811b7a4
--- /dev/null
+++ b/tests/integration/batches/test_batches.py
@@ -0,0 +1,270 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Integration tests for the Llama Stack batch processing functionality.
+
+This module contains comprehensive integration tests for the batch processing API,
+using the OpenAI-compatible client interface for consistency.
+
+Test Categories:
+    1. Core Batch Operations:
+        - test_batch_creation_and_retrieval: Comprehensive batch creation, structure validation, and retrieval
+        - test_batch_listing: Basic batch listing functionality
+        - test_batch_immediate_cancellation: Batch cancellation workflow
+        # TODO: cancel during processing
+
+    2. End-to-End Processing:
+        - test_batch_e2e_chat_completions: Full chat completions workflow with output and error validation
+
+Note: Error conditions and edge cases are primarily tested in test_batches_errors.py
+for better organization and separation of concerns.
+
+CLEANUP WARNING: These tests currently create batches that are not automatically
+cleaned up after test completion. This may lead to resource accumulation over
+multiple test runs. Only test_batch_immediate_cancellation properly cancels its batch.
+The test_batch_e2e_chat_completions test does clean up its output and error files.
+"""
+
+import json
+
+
+class TestBatchesIntegration:
+    """Integration tests for the batches API."""
+
+    def test_batch_creation_and_retrieval(self, openai_client, batch_helper, text_model_id):
+        """Test comprehensive batch creation and retrieval scenarios."""
+        test_metadata = {
+            "test_type": "comprehensive",
+            "purpose": "creation_and_retrieval_test",
+            "version": "1.0",
+            "tags": "test,batch",
+        }
+
+        batch_requests = [
+            {
+                "custom_id": "request-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "batch_creation_test") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata=test_metadata,
+            )
+
+            assert batch.endpoint == "/v1/chat/completions"
+            assert batch.input_file_id == uploaded_file.id
+            assert batch.completion_window == "24h"
+            assert batch.metadata == test_metadata
+
+            retrieved_batch = openai_client.batches.retrieve(batch.id)
+
+            assert retrieved_batch.id == batch.id
+            assert retrieved_batch.object == batch.object
+            assert retrieved_batch.endpoint == batch.endpoint
+            assert retrieved_batch.input_file_id == batch.input_file_id
+            assert retrieved_batch.completion_window == batch.completion_window
+            assert retrieved_batch.metadata == batch.metadata
+
+    def test_batch_listing(self, openai_client, batch_helper, text_model_id):
+        """
+        Test batch listing.
+
+        This test creates multiple batches and verifies that they can be listed.
+        It also deletes the input files before execution, which means the batches
+        will appear as failed due to missing input files. This is expected and
+        a good thing, because it means no inference is performed.
+        """
+        batch_ids = []
+
+        for i in range(2):
+            batch_requests = [
+                {
+                    "custom_id": f"request-{i}",
+                    "method": "POST",
+                    "url": "/v1/chat/completions",
+                    "body": {
+                        "model": text_model_id,
+                        "messages": [{"role": "user", "content": f"Hello {i}"}],
+                        "max_tokens": 10,
+                    },
+                }
+            ]
+
+            with batch_helper.create_file(batch_requests, f"batch_input_{i}") as uploaded_file:
+                batch = openai_client.batches.create(
+                    input_file_id=uploaded_file.id,
+                    endpoint="/v1/chat/completions",
+                    completion_window="24h",
+                )
+                batch_ids.append(batch.id)
+
+        batch_list = openai_client.batches.list()
+
+        assert isinstance(batch_list.data, list)
+
+        listed_batch_ids = {b.id for b in batch_list.data}
+        for batch_id in batch_ids:
+            assert batch_id in listed_batch_ids
+
+    def test_batch_immediate_cancellation(self, openai_client, batch_helper, text_model_id):
+        """Test immediate batch cancellation."""
+        batch_requests = [
+            {
+                "custom_id": "request-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            # hopefully cancel the batch before it completes
+            cancelling_batch = openai_client.batches.cancel(batch.id)
+            assert cancelling_batch.status in ["cancelling", "cancelled"]
+            assert isinstance(cancelling_batch.cancelling_at, int), (
+                f"cancelling_at should be int, got {type(cancelling_batch.cancelling_at)}"
+            )
+
+            final_batch = batch_helper.wait_for(
+                batch.id,
+                max_wait_time=3 * 60,  # often takes 10-11 minutes, give it 3 min
+                expected_statuses={"cancelled"},
+                timeout_action="skip",
+            )
+
+        assert final_batch.status == "cancelled"
+        assert isinstance(final_batch.cancelled_at, int), (
+            f"cancelled_at should be int, got {type(final_batch.cancelled_at)}"
+        )
+
+    def test_batch_e2e_chat_completions(self, openai_client, batch_helper, text_model_id):
+        """Test end-to-end batch processing for chat completions with both successful and failed operations."""
+        batch_requests = [
+            {
+                "custom_id": "success-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Say hello"}],
+                    "max_tokens": 20,
+                },
+            },
+            {
+                "custom_id": "error-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"rolez": "user", "contentz": "This should fail"}],  # Invalid keys to trigger error
+                    # note: ollama does not validate max_tokens values or the "role" key, so they won't trigger an error
+                },
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata={"test": "e2e_success_and_errors_test"},
+            )
+
+            final_batch = batch_helper.wait_for(
+                batch.id,
+                max_wait_time=3 * 60,  # often takes 2-3 minutes
+                expected_statuses={"completed"},
+                timeout_action="skip",
+            )
+
+        # Expecting a completed batch with both successful and failed requests
+        #  Batch(id='batch_xxx',
+        #        completion_window='24h',
+        #        created_at=...,
+        #        endpoint='/v1/chat/completions',
+        #        input_file_id='file-xxx',
+        #        object='batch',
+        #        status='completed',
+        #        output_file_id='file-xxx',
+        #        error_file_id='file-xxx',
+        #        request_counts=BatchRequestCounts(completed=1, failed=1, total=2))
+
+        assert final_batch.status == "completed"
+        assert final_batch.request_counts is not None
+        assert final_batch.request_counts.total == 2
+        assert final_batch.request_counts.completed == 1
+        assert final_batch.request_counts.failed == 1
+
+        assert final_batch.output_file_id is not None, "Output file should exist for successful requests"
+
+        output_content = openai_client.files.content(final_batch.output_file_id)
+        if isinstance(output_content, str):
+            output_text = output_content
+        else:
+            output_text = output_content.content.decode("utf-8")
+
+        output_lines = output_text.strip().split("\n")
+
+        for line in output_lines:
+            result = json.loads(line)
+
+            assert "id" in result
+            assert "custom_id" in result
+            assert result["custom_id"] == "success-1"
+
+            assert "response" in result
+
+            assert result["response"]["status_code"] == 200
+            assert "body" in result["response"]
+            assert "choices" in result["response"]["body"]
+
+        assert final_batch.error_file_id is not None, "Error file should exist for failed requests"
+
+        error_content = openai_client.files.content(final_batch.error_file_id)
+        if isinstance(error_content, str):
+            error_text = error_content
+        else:
+            error_text = error_content.content.decode("utf-8")
+
+        error_lines = error_text.strip().split("\n")
+
+        for line in error_lines:
+            result = json.loads(line)
+
+            assert "id" in result
+            assert "custom_id" in result
+            assert result["custom_id"] == "error-1"
+            assert "error" in result
+            error = result["error"]
+            assert error is not None
+            assert "code" in error or "message" in error, "Error should have code or message"
+
+        deleted_output_file = openai_client.files.delete(final_batch.output_file_id)
+        assert deleted_output_file.deleted, f"Output file {final_batch.output_file_id} was not deleted successfully"
+
+        deleted_error_file = openai_client.files.delete(final_batch.error_file_id)
+        assert deleted_error_file.deleted, f"Error file {final_batch.error_file_id} was not deleted successfully"
diff --git a/tests/integration/batches/test_batches_errors.py b/tests/integration/batches/test_batches_errors.py
new file mode 100644
index 000000000..bc94a182e
--- /dev/null
+++ b/tests/integration/batches/test_batches_errors.py
@@ -0,0 +1,693 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Error handling and edge case tests for the Llama Stack batch processing functionality.
+
+This module focuses exclusively on testing error conditions, validation failures,
+and edge cases for batch operations to ensure robust error handling and graceful
+degradation.
+
+Test Categories:
+    1. File and Input Validation:
+        - test_batch_nonexistent_file_id: Handling invalid file IDs
+        - test_batch_malformed_jsonl: Processing malformed JSONL input files
+        - test_file_malformed_batch_file: Handling malformed files at upload time
+        - test_batch_missing_required_fields: Validation of required request fields
+
+    2. API Endpoint and Model Validation:
+        - test_batch_invalid_endpoint: Invalid endpoint handling during creation
+        - test_batch_error_handling_invalid_model: Error handling with nonexistent models
+        - test_batch_endpoint_mismatch: Validation of endpoint/URL consistency
+
+    3. Batch Lifecycle Error Handling:
+        - test_batch_retrieve_nonexistent: Retrieving non-existent batches
+        - test_batch_cancel_nonexistent: Cancelling non-existent batches
+        - test_batch_cancel_completed: Attempting to cancel completed batches
+
+    4. Parameter and Configuration Validation:
+        - test_batch_invalid_completion_window: Invalid completion window values
+        - test_batch_invalid_metadata_types: Invalid metadata type validation
+        - test_batch_missing_required_body_fields: Validation of required fields in request body
+
+    5. Feature Restriction and Compatibility:
+        - test_batch_streaming_not_supported: Streaming request rejection
+        - test_batch_mixed_streaming_requests: Mixed streaming/non-streaming validation
+
+Note: Core functionality and OpenAI compatibility tests are located in
+test_batches_integration.py for better organization and separation of concerns.
+
+CLEANUP WARNING: These tests create batches to test error conditions but do not
+automatically clean them up after test completion. While most error tests create
+batches that fail quickly, some may create valid batches that consume resources.
+"""
+
+import pytest
+from openai import BadRequestError, ConflictError, NotFoundError
+
+
+class TestBatchesErrorHandling:
+    """Error handling and edge case tests for the batches API using OpenAI client."""
+
+    def test_batch_nonexistent_file_id(self, openai_client, batch_helper):
+        """Test batch creation with nonexistent input file ID."""
+
+        batch = openai_client.batches.create(
+            input_file_id="file-nonexistent-xyz",
+            endpoint="/v1/chat/completions",
+            completion_window="24h",
+        )
+
+        final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='invalid_request',
+        #        line=None,
+        #        message='Cannot find file ..., or organization ... does not have access to it.',
+        #        param='file_id')
+        #    ], object='list'),
+        #    failed_at=1754566971,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.code == "invalid_request"
+        assert "cannot find file" in error.message.lower()
+
+    def test_batch_invalid_endpoint(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with invalid endpoint."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-endpoint",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            with pytest.raises(BadRequestError) as exc_info:
+                openai_client.batches.create(
+                    input_file_id=uploaded_file.id,
+                    endpoint="/v1/invalid/endpoint",
+                    completion_window="24h",
+                )
+
+            # Expected -
+            #  Error code: 400 - {
+            #    'error': {
+            #      'message': "Invalid value: '/v1/invalid/endpoint'. Supported values are: '/v1/chat/completions', '/v1/completions', '/v1/embeddings', and '/v1/responses'.",
+            #      'type': 'invalid_request_error',
+            #      'param': 'endpoint',
+            #      'code': 'invalid_value'
+            #    }
+            #  }
+
+            error_msg = str(exc_info.value).lower()
+            assert exc_info.value.status_code == 400
+            assert "invalid value" in error_msg
+            assert "/v1/invalid/endpoint" in error_msg
+            assert "supported values" in error_msg
+            assert "endpoint" in error_msg
+            assert "invalid_value" in error_msg
+
+    def test_batch_malformed_jsonl(self, openai_client, batch_helper):
+        """
+        Test batch with malformed JSONL input.
+
+        The /v1/files endpoint requires valid JSONL format, so we provide a well formed line
+        before a malformed line to ensure we get to the /v1/batches validation stage.
+        """
+        with batch_helper.create_file(
+            """{"custom_id": "valid", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "test"}}
+{invalid json here""",
+            "malformed_batch_input.jsonl",
+        ) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      ...,
+        #      BatchError(code='invalid_json_line',
+        #                 line=2,
+        #                 message='This line is not parseable as valid JSON.',
+        #                 param=None)
+        #    ], object='list'),
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) > 0
+        error = final_batch.errors.data[-1]  # get last error because first may be about the "test" model
+        assert error.code == "invalid_json_line"
+        assert error.line == 2
+        assert "not" in error.message.lower()
+        assert "valid json" in error.message.lower()
+
+    @pytest.mark.xfail(reason="Not all file providers validate content")
+    @pytest.mark.parametrize("batch_requests", ["", "{malformed json"], ids=["empty", "malformed"])
+    def test_file_malformed_batch_file(self, openai_client, batch_helper, batch_requests):
+        """Test file upload with malformed content."""
+
+        with pytest.raises(BadRequestError) as exc_info:
+            with batch_helper.create_file(batch_requests, "malformed_batch_input_file.jsonl"):
+                # /v1/files rejects the file, we don't get to batch creation
+                pass
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 400
+        assert "invalid file format" in error_msg
+        assert "jsonl" in error_msg
+
+    def test_batch_retrieve_nonexistent(self, openai_client):
+        """Test retrieving nonexistent batch."""
+        with pytest.raises(NotFoundError) as exc_info:
+            openai_client.batches.retrieve("batch-nonexistent-xyz")
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 404
+        assert "no batch found" in error_msg or "not found" in error_msg
+
+    def test_batch_cancel_nonexistent(self, openai_client):
+        """Test cancelling nonexistent batch."""
+        with pytest.raises(NotFoundError) as exc_info:
+            openai_client.batches.cancel("batch-nonexistent-xyz")
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 404
+        assert "no batch found" in error_msg or "not found" in error_msg
+
+    def test_batch_cancel_completed(self, openai_client, batch_helper, text_model_id):
+        """Test cancelling already completed batch."""
+        batch_requests = [
+            {
+                "custom_id": "cancel-completed",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Quick test"}],
+                    "max_tokens": 5,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "cancel_test_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(
+                batch.id,
+                max_wait_time=3 * 60,  # often take 10-11 min, give it 3 min
+                expected_statuses={"completed"},
+                timeout_action="skip",
+            )
+
+        deleted_file = openai_client.files.delete(final_batch.output_file_id)
+        assert deleted_file.deleted, f"File {final_batch.output_file_id} was not deleted successfully"
+
+        with pytest.raises(ConflictError) as exc_info:
+            openai_client.batches.cancel(batch.id)
+
+        # Expecting -
+        #   Error code: 409 - {
+        #     'error': {
+        #       'message': "Cannot cancel a batch with status 'completed'.",
+        #       'type': 'invalid_request_error',
+        #       'param': None,
+        #       'code': None
+        #     }
+        #   }
+        #
+        # NOTE: Same for "failed", cancelling "cancelled" batches is allowed
+
+        error_msg = str(exc_info.value).lower()
+        assert exc_info.value.status_code == 409
+        assert "cannot cancel" in error_msg
+
+    def test_batch_missing_required_fields(self, openai_client, batch_helper, text_model_id):
+        """Test batch with requests missing required fields."""
+        batch_requests = [
+            {
+                # Missing custom_id
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "No custom_id"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "no-method",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "No method"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "no-url",
+                "method": "POST",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "No URL"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "no-body",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests, "missing_fields_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(
+        #      data=[
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=1,
+        #          message="Missing required parameter: 'custom_id'.",
+        #          param='custom_id'
+        #        ),
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=2,
+        #          message="Missing required parameter: 'method'.",
+        #          param='method'
+        #        ),
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=3,
+        #          message="Missing required parameter: 'url'.",
+        #          param='url'
+        #        ),
+        #        BatchError(
+        #          code='missing_required_parameter',
+        #          line=4,
+        #          message="Missing required parameter: 'body'.",
+        #          param='body'
+        #        )
+        #    ], object='list'),
+        #    failed_at=1754566945,
+        #    ...)
+        #  )
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 4
+        no_custom_id_error = final_batch.errors.data[0]
+        assert no_custom_id_error.code == "missing_required_parameter"
+        assert no_custom_id_error.line == 1
+        assert "missing" in no_custom_id_error.message.lower()
+        assert "custom_id" in no_custom_id_error.message.lower()
+        no_method_error = final_batch.errors.data[1]
+        assert no_method_error.code == "missing_required_parameter"
+        assert no_method_error.line == 2
+        assert "missing" in no_method_error.message.lower()
+        assert "method" in no_method_error.message.lower()
+        no_url_error = final_batch.errors.data[2]
+        assert no_url_error.code == "missing_required_parameter"
+        assert no_url_error.line == 3
+        assert "missing" in no_url_error.message.lower()
+        assert "url" in no_url_error.message.lower()
+        no_body_error = final_batch.errors.data[3]
+        assert no_body_error.code == "missing_required_parameter"
+        assert no_body_error.line == 4
+        assert "missing" in no_body_error.message.lower()
+        assert "body" in no_body_error.message.lower()
+
+    def test_batch_invalid_completion_window(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with invalid completion window."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-completion-window",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            for window in ["1h", "48h", "invalid", ""]:
+                with pytest.raises(BadRequestError) as exc_info:
+                    openai_client.batches.create(
+                        input_file_id=uploaded_file.id,
+                        endpoint="/v1/chat/completions",
+                        completion_window=window,
+                    )
+            assert exc_info.value.status_code == 400
+            error_msg = str(exc_info.value).lower()
+            assert "error" in error_msg
+            assert "completion_window" in error_msg
+
+    def test_batch_streaming_not_supported(self, openai_client, batch_helper, text_model_id):
+        """Test that streaming responses are not supported in batches."""
+        batch_requests = [
+            {
+                "custom_id": "streaming-test",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                    "stream": True,  # Not supported
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "streaming_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #       BatchError(code='streaming_unsupported',
+        #         line=1,
+        #         message='Chat Completions: Streaming is not supported in the Batch API.',
+        #         param='body.stream')
+        #    ], object='list'),
+        #    failed_at=1754566965,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.code == "streaming_unsupported"
+        assert error.line == 1
+        assert "streaming" in error.message.lower()
+        assert "not supported" in error.message.lower()
+        assert error.param == "body.stream"
+        assert final_batch.failed_at is not None
+
+    def test_batch_mixed_streaming_requests(self, openai_client, batch_helper, text_model_id):
+        """
+        Test batch with mixed streaming and non-streaming requests.
+
+        This is distinct from test_batch_streaming_not_supported, which tests a single
+        streaming request, to ensure an otherwise valid batch fails when a single
+        streaming request is included.
+        """
+        batch_requests = [
+            {
+                "custom_id": "valid-non-streaming-request",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello without streaming"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "streaming-request",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello with streaming"}],
+                    "max_tokens": 10,
+                    "stream": True,  # Not supported
+                },
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests, "mixed_streaming_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='streaming_unsupported',
+        #        line=2,
+        #        message='Chat Completions: Streaming is not supported in the Batch API.',
+        #        param='body.stream')
+        #    ], object='list'),
+        #    failed_at=1754574442,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.code == "streaming_unsupported"
+        assert error.line == 2
+        assert "streaming" in error.message.lower()
+        assert "not supported" in error.message.lower()
+        assert error.param == "body.stream"
+        assert final_batch.failed_at is not None
+
+    def test_batch_endpoint_mismatch(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with mismatched endpoint and request URL."""
+        batch_requests = [
+            {
+                "custom_id": "endpoint-mismatch",
+                "method": "POST",
+                "url": "/v1/embeddings",  # Different from batch endpoint
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests, "endpoint_mismatch_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",  # Different from request URL
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='invalid_url',
+        #        line=1,
+        #        message='The URL provided for this request does not match the batch endpoint.',
+        #        param='url')
+        #    ], object='list'),
+        #    failed_at=1754566972,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.line == 1
+        assert error.code == "invalid_url"
+        assert "does not match" in error.message.lower()
+        assert "endpoint" in error.message.lower()
+        assert final_batch.failed_at is not None
+
+    def test_batch_error_handling_invalid_model(self, openai_client, batch_helper):
+        """Test batch error handling with invalid model."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-model",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": "nonexistent-model-xyz",
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(code='model_not_found',
+        #        line=1,
+        #        message="The provided model 'nonexistent-model-xyz' is not supported by the Batch API.",
+        #        param='body.model')
+        #    ], object='list'),
+        #    failed_at=1754566978,
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 1
+        error = final_batch.errors.data[0]
+        assert error.line == 1
+        assert error.code == "model_not_found"
+        assert "not supported" in error.message.lower()
+        assert error.param == "body.model"
+        assert final_batch.failed_at is not None
+
+    def test_batch_missing_required_body_fields(self, openai_client, batch_helper, text_model_id):
+        """Test batch with requests missing required fields in body (model and messages)."""
+        batch_requests = [
+            {
+                "custom_id": "missing-model",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    # Missing model field
+                    "messages": [{"role": "user", "content": "Hello without model"}],
+                    "max_tokens": 10,
+                },
+            },
+            {
+                "custom_id": "missing-messages",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    # Missing messages field
+                    "max_tokens": 10,
+                },
+            },
+        ]
+
+        with batch_helper.create_file(batch_requests, "missing_body_fields_batch_input") as uploaded_file:
+            batch = openai_client.batches.create(
+                input_file_id=uploaded_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            final_batch = batch_helper.wait_for(batch.id, expected_statuses={"failed"})
+
+        # Expecting -
+        #  Batch(...,
+        #    status='failed',
+        #    errors=Errors(data=[
+        #      BatchError(
+        #        code='invalid_request',
+        #        line=1,
+        #        message='Model parameter is required.',
+        #        param='body.model'),
+        #      BatchError(
+        #        code='invalid_request',
+        #        line=2,
+        #        message='Messages parameter is required.',
+        #        param='body.messages')
+        #      ], object='list'),
+        #    ...)
+
+        assert final_batch.status == "failed"
+        assert final_batch.errors is not None
+        assert len(final_batch.errors.data) == 2
+
+        model_error = final_batch.errors.data[0]
+        assert model_error.line == 1
+        assert "model" in model_error.message.lower()
+        assert model_error.param == "body.model"
+
+        messages_error = final_batch.errors.data[1]
+        assert messages_error.line == 2
+        assert "messages" in messages_error.message.lower()
+        assert messages_error.param == "body.messages"
+
+        assert final_batch.failed_at is not None
+
+    def test_batch_invalid_metadata_types(self, openai_client, batch_helper, text_model_id):
+        """Test batch creation with invalid metadata types (like lists)."""
+        batch_requests = [
+            {
+                "custom_id": "invalid-metadata-type",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": text_model_id,
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        with batch_helper.create_file(batch_requests) as uploaded_file:
+            with pytest.raises(Exception) as exc_info:
+                openai_client.batches.create(
+                    input_file_id=uploaded_file.id,
+                    endpoint="/v1/chat/completions",
+                    completion_window="24h",
+                    metadata={
+                        "tags": ["tag1", "tag2"],  # Invalid type, should be a string
+                    },
+                )
+
+        # Expecting -
+        #  Error code: 400 - {'error':
+        #    {'message': "Invalid type for 'metadata.tags': expected a string,
+        #                 but got an array instead.",
+        #     'type': 'invalid_request_error', 'param': 'metadata.tags',
+        #     'code': 'invalid_type'}}
+
+        error_msg = str(exc_info.value).lower()
+        assert "400" in error_msg
+        assert "tags" in error_msg
+        assert "string" in error_msg
diff --git a/tests/integration/recordings/index.sqlite b/tests/integration/recordings/index.sqlite
index e01c8803aba178128bbcf05b06a5c949fcd928b1..7b6eb6a67119bdc6ebce3539c5224bb412ec8a6e 100644
GIT binary patch
delta 611
zcmZozz}#?vd4e=+FarYv-$Vs_#^8+!OZdgu`L6*582I?#@qgie!T*c@5&s?jYn!D6
z_VBX-#d*2{Hs6ugaA4G%%$i`Yz@^KxhQEj>l}C={1y>2P9n*K#GRE61!rY$ROkBI!
zST?65d}U_ycVlVfWt!~Z<}i7FiU*_5<`*eCEQ~T63ndsQD;#I7H%>N4O0!H#v@lFE
zvM{quHA=EHvrJ4jG&4;x0m_+~SQsZKTbLyo8(Sn=7?~NEm?c}LSfrU-n5U!~Sz4sA
z27xUwF-bH`u}n!dFf%YpGO#oU8fXC2lVp-=oM>Q?YLS$bWNe<C1~k*uz`)$p)Y#I*
zG9}H_h|4h1&@|D^Ac-|_^X{x_R}P*Uwj{P&+yxU0<u=D`;9z6)nOrT9Az-0vXsTdj
zXk}_@WnfX-<v;oEwmfbqw^69Wb@Cjc*2(jC@@#&-y_=acfTNLDUz*pLW3v5OhsnEl
ztFVKW8X9hXwtJQ!W5DLdv-M0oyudJJ;ycE`pUig*7#@5R8y1B(N_GW-G@IHRb5I&I
kNctxSY_yAqIN!t&Xh$H-4syc{7HmlRL3VIJ>@YF}0DR%DfB*mh

delta 248
zcmZoTz}&Ead4e>nD+2=q??eTAM%RrAOZdgu`0oG(7<l;K@qgie!T*c@5&s=-zs*tt
zd-x|Oh_Y?IBd_7Ws56;0!Crx94Sx|&Dvun?3uZf}@2q8vw^@X_J-L~<cC)cC>}S>A
zoRaXBna#<KrID9uvV)t$<oPKcjP{#fr0B3PuG(0*jB&EUan{WTvdUaJ*pk?8aToB^
zOe~b!9J7IgjnRH`wLpe|g|4Bgf{}rhk%^U|acP&+<h$GQxS`xep$^x{bA(za&)><j
t`T6#4W=?00MqYhsUSp2Q_GcX?@7}G#4pC~p`PuGSf{f0a8_(7=0RVoKPVoQ$

diff --git a/tests/integration/recordings/responses/3c0bf9ba81b2.json b/tests/integration/recordings/responses/3c0bf9ba81b2.json
new file mode 100644
index 000000000..1b5f16c22
--- /dev/null
+++ b/tests/integration/recordings/responses/3c0bf9ba81b2.json
@@ -0,0 +1,56 @@
+{
+  "request": {
+    "method": "POST",
+    "url": "http://0.0.0.0:11434/v1/v1/chat/completions",
+    "headers": {},
+    "body": {
+      "model": "llama3.2:3b-instruct-fp16",
+      "messages": [
+        {
+          "role": "user",
+          "content": "Quick test"
+        }
+      ],
+      "max_tokens": 5
+    },
+    "endpoint": "/v1/chat/completions",
+    "model": "llama3.2:3b-instruct-fp16"
+  },
+  "response": {
+    "body": {
+      "__type__": "openai.types.chat.chat_completion.ChatCompletion",
+      "__data__": {
+        "id": "chatcmpl-651",
+        "choices": [
+          {
+            "finish_reason": "length",
+            "index": 0,
+            "logprobs": null,
+            "message": {
+              "content": "I'm ready to help",
+              "refusal": null,
+              "role": "assistant",
+              "annotations": null,
+              "audio": null,
+              "function_call": null,
+              "tool_calls": null
+            }
+          }
+        ],
+        "created": 1755294941,
+        "model": "llama3.2:3b-instruct-fp16",
+        "object": "chat.completion",
+        "service_tier": null,
+        "system_fingerprint": "fp_ollama",
+        "usage": {
+          "completion_tokens": 5,
+          "prompt_tokens": 27,
+          "total_tokens": 32,
+          "completion_tokens_details": null,
+          "prompt_tokens_details": null
+        }
+      }
+    },
+    "is_streaming": false
+  }
+}
diff --git a/tests/integration/recordings/responses/44a1d9de0602.json b/tests/integration/recordings/responses/44a1d9de0602.json
new file mode 100644
index 000000000..2d158a06c
--- /dev/null
+++ b/tests/integration/recordings/responses/44a1d9de0602.json
@@ -0,0 +1,56 @@
+{
+  "request": {
+    "method": "POST",
+    "url": "http://0.0.0.0:11434/v1/v1/chat/completions",
+    "headers": {},
+    "body": {
+      "model": "llama3.2:3b-instruct-fp16",
+      "messages": [
+        {
+          "role": "user",
+          "content": "Say hello"
+        }
+      ],
+      "max_tokens": 20
+    },
+    "endpoint": "/v1/chat/completions",
+    "model": "llama3.2:3b-instruct-fp16"
+  },
+  "response": {
+    "body": {
+      "__type__": "openai.types.chat.chat_completion.ChatCompletion",
+      "__data__": {
+        "id": "chatcmpl-987",
+        "choices": [
+          {
+            "finish_reason": "length",
+            "index": 0,
+            "logprobs": null,
+            "message": {
+              "content": "Hello! It's nice to meet you. Is there something I can help you with or would you",
+              "refusal": null,
+              "role": "assistant",
+              "annotations": null,
+              "audio": null,
+              "function_call": null,
+              "tool_calls": null
+            }
+          }
+        ],
+        "created": 1755294921,
+        "model": "llama3.2:3b-instruct-fp16",
+        "object": "chat.completion",
+        "service_tier": null,
+        "system_fingerprint": "fp_ollama",
+        "usage": {
+          "completion_tokens": 20,
+          "prompt_tokens": 27,
+          "total_tokens": 47,
+          "completion_tokens_details": null,
+          "prompt_tokens_details": null
+        }
+      }
+    },
+    "is_streaming": false
+  }
+}
diff --git a/tests/integration/recordings/responses/4a3a4447b16b.json b/tests/integration/recordings/responses/4a3a4447b16b.json
index ee1ee6d70..a31c583c7 100644
--- a/tests/integration/recordings/responses/4a3a4447b16b.json
+++ b/tests/integration/recordings/responses/4a3a4447b16b.json
@@ -14,7 +14,7 @@
         "models": [
           {
             "model": "nomic-embed-text:latest",
-            "modified_at": "2025-08-15T20:24:13.254634Z",
+            "modified_at": "2025-08-15T21:55:08.088554Z",
             "digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f",
             "size": 274302450,
             "details": {
diff --git a/tests/unit/providers/batches/test_reference.py b/tests/unit/providers/batches/test_reference.py
new file mode 100644
index 000000000..9fe0cc710
--- /dev/null
+++ b/tests/unit/providers/batches/test_reference.py
@@ -0,0 +1,753 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Test suite for the reference implementation of the Batches API.
+
+The tests are categorized and outlined below, keep this updated:
+
+- Batch creation with various parameters and validation:
+  * test_create_and_retrieve_batch_success (positive)
+  * test_create_batch_without_metadata (positive)
+  * test_create_batch_completion_window (negative)
+  * test_create_batch_invalid_endpoints (negative)
+  * test_create_batch_invalid_metadata (negative)
+
+- Batch retrieval and error handling for non-existent batches:
+  * test_retrieve_batch_not_found (negative)
+
+- Batch cancellation with proper status transitions:
+  * test_cancel_batch_success (positive)
+  * test_cancel_batch_invalid_statuses (negative)
+  * test_cancel_batch_not_found (negative)
+
+- Batch listing with pagination and filtering:
+  * test_list_batches_empty (positive)
+  * test_list_batches_single_batch (positive)
+  * test_list_batches_multiple_batches (positive)
+  * test_list_batches_with_limit (positive)
+  * test_list_batches_with_pagination (positive)
+  * test_list_batches_invalid_after (negative)
+
+- Data persistence in the underlying key-value store:
+  * test_kvstore_persistence (positive)
+
+- Batch processing concurrency control:
+  * test_max_concurrent_batches (positive)
+
+- Input validation testing (direct _validate_input method tests):
+  * test_validate_input_file_not_found (negative)
+  * test_validate_input_file_exists_empty_content (positive)
+  * test_validate_input_file_mixed_valid_invalid_json (mixed)
+  * test_validate_input_invalid_model (negative)
+  * test_validate_input_url_mismatch (negative)
+  * test_validate_input_multiple_errors_per_request (negative)
+  * test_validate_input_invalid_request_format (negative)
+  * test_validate_input_missing_parameters (parametrized negative - custom_id, method, url, body, model, messages missing validation)
+  * test_validate_input_invalid_parameter_types (parametrized negative - custom_id, url, method, body, model, messages type validation)
+
+The tests use temporary SQLite databases for isolation and mock external
+dependencies like inference, files, and models APIs.
+"""
+
+import json
+import tempfile
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from llama_stack.apis.batches import BatchObject
+from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError
+from llama_stack.providers.inline.batches.reference.batches import ReferenceBatchesImpl
+from llama_stack.providers.inline.batches.reference.config import ReferenceBatchesImplConfig
+from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig
+
+
+class TestReferenceBatchesImpl:
+    """Test the reference implementation of the Batches API."""
+
+    @pytest.fixture
+    async def provider(self):
+        """Create a test provider instance with temporary database."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            db_path = Path(tmpdir) / "test_batches.db"
+            kvstore_config = SqliteKVStoreConfig(db_path=str(db_path))
+            config = ReferenceBatchesImplConfig(kvstore=kvstore_config)
+
+            # Create kvstore and mock APIs
+            from unittest.mock import AsyncMock
+
+            from llama_stack.providers.utils.kvstore import kvstore_impl
+
+            kvstore = await kvstore_impl(config.kvstore)
+            mock_inference = AsyncMock()
+            mock_files = AsyncMock()
+            mock_models = AsyncMock()
+
+            provider = ReferenceBatchesImpl(config, mock_inference, mock_files, mock_models, kvstore)
+            await provider.initialize()
+
+            # unit tests should not require background processing
+            provider.process_batches = False
+
+            yield provider
+
+            await provider.shutdown()
+
+    @pytest.fixture
+    def sample_batch_data(self):
+        """Sample batch data for testing."""
+        return {
+            "input_file_id": "file_abc123",
+            "endpoint": "/v1/chat/completions",
+            "completion_window": "24h",
+            "metadata": {"test": "true", "priority": "high"},
+        }
+
+    def _validate_batch_type(self, batch, expected_metadata=None):
+        """
+        Helper function to validate batch object structure and field types.
+
+        Note: This validates the direct BatchObject from the provider, not the
+              client library response which has a different structure.
+
+        Args:
+            batch: The BatchObject instance to validate.
+            expected_metadata: Optional expected metadata dictionary to validate against.
+        """
+        assert isinstance(batch.id, str)
+        assert isinstance(batch.completion_window, str)
+        assert isinstance(batch.created_at, int)
+        assert isinstance(batch.endpoint, str)
+        assert isinstance(batch.input_file_id, str)
+        assert batch.object == "batch"
+        assert batch.status in [
+            "validating",
+            "failed",
+            "in_progress",
+            "finalizing",
+            "completed",
+            "expired",
+            "cancelling",
+            "cancelled",
+        ]
+
+        if expected_metadata is not None:
+            assert batch.metadata == expected_metadata
+
+        timestamp_fields = [
+            "cancelled_at",
+            "cancelling_at",
+            "completed_at",
+            "expired_at",
+            "expires_at",
+            "failed_at",
+            "finalizing_at",
+            "in_progress_at",
+        ]
+        for field in timestamp_fields:
+            field_value = getattr(batch, field, None)
+            if field_value is not None:
+                assert isinstance(field_value, int), f"{field} should be int or None, got {type(field_value)}"
+
+        file_id_fields = ["error_file_id", "output_file_id"]
+        for field in file_id_fields:
+            field_value = getattr(batch, field, None)
+            if field_value is not None:
+                assert isinstance(field_value, str), f"{field} should be str or None, got {type(field_value)}"
+
+        if hasattr(batch, "request_counts") and batch.request_counts is not None:
+            assert isinstance(batch.request_counts.completed, int), (
+                f"request_counts.completed should be int, got {type(batch.request_counts.completed)}"
+            )
+            assert isinstance(batch.request_counts.failed, int), (
+                f"request_counts.failed should be int, got {type(batch.request_counts.failed)}"
+            )
+            assert isinstance(batch.request_counts.total, int), (
+                f"request_counts.total should be int, got {type(batch.request_counts.total)}"
+            )
+
+        if hasattr(batch, "errors") and batch.errors is not None:
+            assert isinstance(batch.errors, dict), f"errors should be object or dict, got {type(batch.errors)}"
+
+            if hasattr(batch.errors, "data") and batch.errors.data is not None:
+                assert isinstance(batch.errors.data, list), (
+                    f"errors.data should be list or None, got {type(batch.errors.data)}"
+                )
+
+                for i, error_item in enumerate(batch.errors.data):
+                    assert isinstance(error_item, dict), (
+                        f"errors.data[{i}] should be object or dict, got {type(error_item)}"
+                    )
+
+                    if hasattr(error_item, "code") and error_item.code is not None:
+                        assert isinstance(error_item.code, str), (
+                            f"errors.data[{i}].code should be str or None, got {type(error_item.code)}"
+                        )
+
+                    if hasattr(error_item, "line") and error_item.line is not None:
+                        assert isinstance(error_item.line, int), (
+                            f"errors.data[{i}].line should be int or None, got {type(error_item.line)}"
+                        )
+
+                    if hasattr(error_item, "message") and error_item.message is not None:
+                        assert isinstance(error_item.message, str), (
+                            f"errors.data[{i}].message should be str or None, got {type(error_item.message)}"
+                        )
+
+                    if hasattr(error_item, "param") and error_item.param is not None:
+                        assert isinstance(error_item.param, str), (
+                            f"errors.data[{i}].param should be str or None, got {type(error_item.param)}"
+                        )
+
+            if hasattr(batch.errors, "object") and batch.errors.object is not None:
+                assert isinstance(batch.errors.object, str), (
+                    f"errors.object should be str or None, got {type(batch.errors.object)}"
+                )
+                assert batch.errors.object == "list", f"errors.object should be 'list', got {batch.errors.object}"
+
+    async def test_create_and_retrieve_batch_success(self, provider, sample_batch_data):
+        """Test successful batch creation and retrieval."""
+        created_batch = await provider.create_batch(**sample_batch_data)
+
+        self._validate_batch_type(created_batch, expected_metadata=sample_batch_data["metadata"])
+
+        assert created_batch.id.startswith("batch_")
+        assert len(created_batch.id) > 13
+        assert created_batch.object == "batch"
+        assert created_batch.endpoint == sample_batch_data["endpoint"]
+        assert created_batch.input_file_id == sample_batch_data["input_file_id"]
+        assert created_batch.completion_window == sample_batch_data["completion_window"]
+        assert created_batch.status == "validating"
+        assert created_batch.metadata == sample_batch_data["metadata"]
+        assert isinstance(created_batch.created_at, int)
+        assert created_batch.created_at > 0
+
+        retrieved_batch = await provider.retrieve_batch(created_batch.id)
+
+        self._validate_batch_type(retrieved_batch, expected_metadata=sample_batch_data["metadata"])
+
+        assert retrieved_batch.id == created_batch.id
+        assert retrieved_batch.input_file_id == created_batch.input_file_id
+        assert retrieved_batch.endpoint == created_batch.endpoint
+        assert retrieved_batch.status == created_batch.status
+        assert retrieved_batch.metadata == created_batch.metadata
+
+    async def test_create_batch_without_metadata(self, provider):
+        """Test batch creation without optional metadata."""
+        batch = await provider.create_batch(
+            input_file_id="file_123", endpoint="/v1/chat/completions", completion_window="24h"
+        )
+
+        assert batch.metadata is None
+
+    async def test_create_batch_completion_window(self, provider):
+        """Test batch creation with invalid completion window."""
+        with pytest.raises(ValueError, match="Invalid completion_window"):
+            await provider.create_batch(
+                input_file_id="file_123", endpoint="/v1/chat/completions", completion_window="now"
+            )
+
+    @pytest.mark.parametrize(
+        "endpoint",
+        [
+            "/v1/embeddings",
+            "/v1/completions",
+            "/v1/invalid/endpoint",
+            "",
+        ],
+    )
+    async def test_create_batch_invalid_endpoints(self, provider, endpoint):
+        """Test batch creation with various invalid endpoints."""
+        with pytest.raises(ValueError, match="Invalid endpoint"):
+            await provider.create_batch(input_file_id="file_123", endpoint=endpoint, completion_window="24h")
+
+    async def test_create_batch_invalid_metadata(self, provider):
+        """Test that batch creation fails with invalid metadata."""
+        with pytest.raises(ValueError, match="should be a valid string"):
+            await provider.create_batch(
+                input_file_id="file_123",
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata={123: "invalid_key"},  # Non-string key
+            )
+
+        with pytest.raises(ValueError, match="should be a valid string"):
+            await provider.create_batch(
+                input_file_id="file_123",
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+                metadata={"valid_key": 456},  # Non-string value
+            )
+
+    async def test_retrieve_batch_not_found(self, provider):
+        """Test error when retrieving non-existent batch."""
+        with pytest.raises(ResourceNotFoundError, match=r"Batch 'nonexistent_batch' not found"):
+            await provider.retrieve_batch("nonexistent_batch")
+
+    async def test_cancel_batch_success(self, provider, sample_batch_data):
+        """Test successful batch cancellation."""
+        created_batch = await provider.create_batch(**sample_batch_data)
+        assert created_batch.status == "validating"
+
+        cancelled_batch = await provider.cancel_batch(created_batch.id)
+
+        assert cancelled_batch.id == created_batch.id
+        assert cancelled_batch.status in ["cancelling", "cancelled"]
+        assert isinstance(cancelled_batch.cancelling_at, int)
+        assert cancelled_batch.cancelling_at >= created_batch.created_at
+
+    @pytest.mark.parametrize("status", ["failed", "expired", "completed"])
+    async def test_cancel_batch_invalid_statuses(self, provider, sample_batch_data, status):
+        """Test error when cancelling batch in final states."""
+        provider.process_batches = False
+        created_batch = await provider.create_batch(**sample_batch_data)
+
+        # directly update status in kvstore
+        await provider._update_batch(created_batch.id, status=status)
+
+        with pytest.raises(ConflictError, match=f"Cannot cancel batch '{created_batch.id}' with status '{status}'"):
+            await provider.cancel_batch(created_batch.id)
+
+    async def test_cancel_batch_not_found(self, provider):
+        """Test error when cancelling non-existent batch."""
+        with pytest.raises(ResourceNotFoundError, match=r"Batch 'nonexistent_batch' not found"):
+            await provider.cancel_batch("nonexistent_batch")
+
+    async def test_list_batches_empty(self, provider):
+        """Test listing batches when none exist."""
+        response = await provider.list_batches()
+
+        assert response.object == "list"
+        assert response.data == []
+        assert response.first_id is None
+        assert response.last_id is None
+        assert response.has_more is False
+
+    async def test_list_batches_single_batch(self, provider, sample_batch_data):
+        """Test listing batches with single batch."""
+        created_batch = await provider.create_batch(**sample_batch_data)
+
+        response = await provider.list_batches()
+
+        assert len(response.data) == 1
+        self._validate_batch_type(response.data[0], expected_metadata=sample_batch_data["metadata"])
+        assert response.data[0].id == created_batch.id
+        assert response.first_id == created_batch.id
+        assert response.last_id == created_batch.id
+        assert response.has_more is False
+
+    async def test_list_batches_multiple_batches(self, provider):
+        """Test listing multiple batches."""
+        batches = [
+            await provider.create_batch(
+                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+            for i in range(3)
+        ]
+
+        response = await provider.list_batches()
+
+        assert len(response.data) == 3
+
+        batch_ids = {batch.id for batch in response.data}
+        expected_ids = {batch.id for batch in batches}
+        assert batch_ids == expected_ids
+        assert response.has_more is False
+
+        assert response.first_id in expected_ids
+        assert response.last_id in expected_ids
+
+    async def test_list_batches_with_limit(self, provider):
+        """Test listing batches with limit parameter."""
+        batches = [
+            await provider.create_batch(
+                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+            for i in range(3)
+        ]
+
+        response = await provider.list_batches(limit=2)
+
+        assert len(response.data) == 2
+        assert response.has_more is True
+        assert response.first_id == response.data[0].id
+        assert response.last_id == response.data[1].id
+        batch_ids = {batch.id for batch in response.data}
+        expected_ids = {batch.id for batch in batches}
+        assert batch_ids.issubset(expected_ids)
+
+    async def test_list_batches_with_pagination(self, provider):
+        """Test listing batches with pagination using 'after' parameter."""
+        for i in range(3):
+            await provider.create_batch(
+                input_file_id=f"file_{i}", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+
+        # Get first page
+        first_page = await provider.list_batches(limit=1)
+        assert len(first_page.data) == 1
+        assert first_page.has_more is True
+
+        # Get second page using 'after'
+        second_page = await provider.list_batches(limit=1, after=first_page.data[0].id)
+        assert len(second_page.data) == 1
+        assert second_page.data[0].id != first_page.data[0].id
+
+        # Verify we got the next batch in order
+        all_batches = await provider.list_batches()
+        expected_second_batch_id = all_batches.data[1].id
+        assert second_page.data[0].id == expected_second_batch_id
+
+    async def test_list_batches_invalid_after(self, provider, sample_batch_data):
+        """Test listing batches with invalid 'after' parameter."""
+        await provider.create_batch(**sample_batch_data)
+
+        response = await provider.list_batches(after="nonexistent_batch")
+
+        # Should return all batches (no filtering when 'after' batch not found)
+        assert len(response.data) == 1
+
+    async def test_kvstore_persistence(self, provider, sample_batch_data):
+        """Test that batches are properly persisted in kvstore."""
+        batch = await provider.create_batch(**sample_batch_data)
+
+        stored_data = await provider.kvstore.get(f"batch:{batch.id}")
+        assert stored_data is not None
+
+        stored_batch_dict = json.loads(stored_data)
+        assert stored_batch_dict["id"] == batch.id
+        assert stored_batch_dict["input_file_id"] == sample_batch_data["input_file_id"]
+
+    async def test_validate_input_file_not_found(self, provider):
+        """Test _validate_input when input file does not exist."""
+        provider.files_api.openai_retrieve_file = AsyncMock(side_effect=Exception("File not found"))
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="nonexistent_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+        assert errors[0].code == "invalid_request"
+        assert errors[0].message == "Cannot find file nonexistent_file."
+        assert errors[0].param == "input_file_id"
+        assert errors[0].line is None
+
+    async def test_validate_input_file_exists_empty_content(self, provider):
+        """Test _validate_input when file exists but is empty."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b""
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="empty_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 0
+        assert len(requests) == 0
+
+    async def test_validate_input_file_mixed_valid_invalid_json(self, provider):
+        """Test _validate_input when file contains valid and invalid JSON lines."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        # Line 1: valid JSON with proper body args, Line 2: invalid JSON
+        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]}}\n{invalid json'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="mixed_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        # Should have 1 JSON parsing error from line 2, and 1 valid request from line 1
+        assert len(errors) == 1
+        assert len(requests) == 1
+
+        assert errors[0].code == "invalid_json_line"
+        assert errors[0].line == 2
+        assert errors[0].message == "This line is not parseable as valid JSON."
+
+        assert requests[0].custom_id == "req-1"
+        assert requests[0].method == "POST"
+        assert requests[0].url == "/v1/chat/completions"
+        assert requests[0].body["model"] == "test-model"
+        assert requests[0].body["messages"] == [{"role": "user", "content": "Hello"}]
+
+    async def test_validate_input_invalid_model(self, provider):
+        """Test _validate_input when file contains request with non-existent model."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "nonexistent-model", "messages": [{"role": "user", "content": "Hello"}]}}'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        provider.models_api.get_model = AsyncMock(side_effect=Exception("Model not found"))
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="invalid_model_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "model_not_found"
+        assert errors[0].line == 1
+        assert errors[0].message == "Model 'nonexistent-model' does not exist or is not supported"
+        assert errors[0].param == "body.model"
+
+    @pytest.mark.parametrize(
+        "param_name,param_path,error_code,error_message",
+        [
+            ("custom_id", "custom_id", "missing_required_parameter", "Missing required parameter: custom_id"),
+            ("method", "method", "missing_required_parameter", "Missing required parameter: method"),
+            ("url", "url", "missing_required_parameter", "Missing required parameter: url"),
+            ("body", "body", "missing_required_parameter", "Missing required parameter: body"),
+            ("model", "body.model", "invalid_request", "Model parameter is required"),
+            ("messages", "body.messages", "invalid_request", "Messages parameter is required"),
+        ],
+    )
+    async def test_validate_input_missing_parameters(self, provider, param_name, param_path, error_code, error_message):
+        """Test _validate_input when file contains request with missing required parameters."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+
+        base_request = {
+            "custom_id": "req-1",
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]},
+        }
+
+        # Remove the specific parameter being tested
+        if "." in param_path:
+            top_level, nested_param = param_path.split(".", 1)
+            del base_request[top_level][nested_param]
+        else:
+            del base_request[param_name]
+
+        mock_response.body = json.dumps(base_request).encode()
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id=f"missing_{param_name}_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == error_code
+        assert errors[0].line == 1
+        assert errors[0].message == error_message
+        assert errors[0].param == param_path
+
+    async def test_validate_input_url_mismatch(self, provider):
+        """Test _validate_input when file contains request with URL that doesn't match batch endpoint."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b'{"custom_id": "req-1", "method": "POST", "url": "/v1/embeddings", "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]}}'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",  # This doesn't match the URL in the request
+            input_file_id="url_mismatch_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "invalid_url"
+        assert errors[0].line == 1
+        assert errors[0].message == "URL provided for this request does not match the batch endpoint"
+        assert errors[0].param == "url"
+
+    async def test_validate_input_multiple_errors_per_request(self, provider):
+        """Test _validate_input when a single request has multiple validation errors."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        # Request missing custom_id, has invalid URL, and missing model in body
+        mock_response.body = (
+            b'{"method": "POST", "url": "/v1/embeddings", "body": {"messages": [{"role": "user", "content": "Hello"}]}}'
+        )
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",  # Doesn't match /v1/embeddings in request
+            input_file_id="multiple_errors_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) >= 2  # At least missing custom_id and URL mismatch
+        assert len(requests) == 0
+
+        for error in errors:
+            assert error.line == 1
+
+        error_codes = {error.code for error in errors}
+        assert "missing_required_parameter" in error_codes  # missing custom_id
+        assert "invalid_url" in error_codes  # URL mismatch
+
+    async def test_validate_input_invalid_request_format(self, provider):
+        """Test _validate_input when file contains non-object JSON (array, string, number)."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+        mock_response.body = b'["not", "a", "request", "object"]'
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id="invalid_format_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "invalid_request"
+        assert errors[0].line == 1
+        assert errors[0].message == "Each line must be a JSON dictionary object"
+
+    @pytest.mark.parametrize(
+        "param_name,param_path,invalid_value,error_message",
+        [
+            ("custom_id", "custom_id", 12345, "Custom_id must be a string"),
+            ("url", "url", 123, "URL must be a string"),
+            ("method", "method", ["POST"], "Method must be a string"),
+            ("body", "body", ["not", "valid"], "Body must be a JSON dictionary object"),
+            ("model", "body.model", 123, "Model must be a string"),
+            ("messages", "body.messages", "invalid messages format", "Messages must be an array"),
+        ],
+    )
+    async def test_validate_input_invalid_parameter_types(
+        self, provider, param_name, param_path, invalid_value, error_message
+    ):
+        """Test _validate_input when file contains request with parameters that have invalid types."""
+        provider.files_api.openai_retrieve_file = AsyncMock()
+        mock_response = MagicMock()
+
+        base_request = {
+            "custom_id": "req-1",
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {"model": "test-model", "messages": [{"role": "user", "content": "Hello"}]},
+        }
+
+        # Override the specific parameter with invalid value
+        if "." in param_path:
+            top_level, nested_param = param_path.split(".", 1)
+            base_request[top_level][nested_param] = invalid_value
+        else:
+            base_request[param_name] = invalid_value
+
+        mock_response.body = json.dumps(base_request).encode()
+        provider.files_api.openai_retrieve_file_content = AsyncMock(return_value=mock_response)
+
+        batch = BatchObject(
+            id="batch_test",
+            object="batch",
+            endpoint="/v1/chat/completions",
+            input_file_id=f"invalid_{param_name}_type_file",
+            completion_window="24h",
+            status="validating",
+            created_at=1234567890,
+        )
+
+        errors, requests = await provider._validate_input(batch)
+
+        assert len(errors) == 1
+        assert len(requests) == 0
+
+        assert errors[0].code == "invalid_request"
+        assert errors[0].line == 1
+        assert errors[0].message == error_message
+        assert errors[0].param == param_path
+
+    async def test_max_concurrent_batches(self, provider):
+        """Test max_concurrent_batches configuration and concurrency control."""
+        import asyncio
+
+        provider._batch_semaphore = asyncio.Semaphore(2)
+
+        provider.process_batches = True  # enable because we're testing background processing
+
+        active_batches = 0
+
+        async def add_and_wait(batch_id: str):
+            nonlocal active_batches
+            active_batches += 1
+            await asyncio.sleep(float("inf"))
+
+        # the first thing done in _process_batch is to acquire the semaphore, then call _process_batch_impl,
+        # so we can replace _process_batch_impl with our mock to control concurrency
+        provider._process_batch_impl = add_and_wait
+
+        for _ in range(3):
+            await provider.create_batch(
+                input_file_id="file_id", endpoint="/v1/chat/completions", completion_window="24h"
+            )
+
+        await asyncio.sleep(0.042)  # let tasks start
+
+        assert active_batches == 2, f"Expected 2 active batches, got {active_batches}"

From 5e7c2250be53c4799a388eb32a8427c5db0dfdee Mon Sep 17 00:00:00 2001
From: Ashwin Bharambe <ashwin.bharambe@gmail.com>
Date: Fri, 15 Aug 2025 16:54:34 -0700
Subject: [PATCH 45/45] test(recording): add a script to schedule recording
 workflow (#3170)

See comment here:
https://github.com/llamastack/llama-stack/pull/3162#issuecomment-3192859097
-- TL;DR it is quite complex to invoke the recording workflow correctly
for an end developer writing tests. This script simplifies the work.

No more manual GitHub UI navigation!

## Script Functionality

  - Auto-detects your current branch and associated PR
  - Finds the right repository context (works from forks!)
  - Runs the workflow where it can actually commit back
  - Validates prerequisites and provides helpful error messages

## How to Use

First ensure you are on the branch which introduced a new test and want
it recorded. **Make sure you have pushed this branch remotely, easiest
is to create a PR.**

```
  # Record tests for current branch
  ./scripts/github/schedule-record-workflow.sh

  # Record specific test subdirectories
  ./scripts/github/schedule-record-workflow.sh --test-subdirs "agents,inference"

  # Record with vision tests enabled
  ./scripts/github/schedule-record-workflow.sh --run-vision-tests

  # Record tests matching a pattern
  ./scripts/github/schedule-record-workflow.sh --test-pattern "test_streaming"
```

## Test Plan

Ran `./scripts/github/schedule-record-workflow.sh -s inference -k
tool_choice` which started
https://github.com/ashwinb/llama-stack/actions/runs/17001534248/job/48204093292
which successfully committed recorded outputs.
---
 .../workflows/record-integration-tests.yml    |  10 +
 scripts/github/schedule-record-workflow.sh    | 279 ++++++++++++++++++
 tests/README.md                               |  33 ++-
 tests/integration/README.md                   |   9 +
 4 files changed, 329 insertions(+), 2 deletions(-)
 create mode 100755 scripts/github/schedule-record-workflow.sh

diff --git a/.github/workflows/record-integration-tests.yml b/.github/workflows/record-integration-tests.yml
index 95403291c..22636f209 100644
--- a/.github/workflows/record-integration-tests.yml
+++ b/.github/workflows/record-integration-tests.yml
@@ -35,6 +35,16 @@ jobs:
       contents: write
 
     steps:
+      - name: Echo workflow inputs
+        run: |
+          echo "::group::Workflow Inputs"
+          echo "test-subdirs: ${{ inputs.test-subdirs }}"
+          echo "test-provider: ${{ inputs.test-provider }}"
+          echo "run-vision-tests: ${{ inputs.run-vision-tests }}"
+          echo "test-pattern: ${{ inputs.test-pattern }}"
+          echo "branch: ${{ github.ref_name }}"
+          echo "::endgroup::"
+
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
diff --git a/scripts/github/schedule-record-workflow.sh b/scripts/github/schedule-record-workflow.sh
new file mode 100755
index 000000000..e381b60b6
--- /dev/null
+++ b/scripts/github/schedule-record-workflow.sh
@@ -0,0 +1,279 @@
+#!/bin/bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+# Script to easily trigger the integration test recording workflow
+# Usage: ./scripts/github/schedule-record-workflow.sh [options]
+
+set -euo pipefail
+
+# Default values
+BRANCH=""
+TEST_SUBDIRS=""
+TEST_PROVIDER="ollama"
+RUN_VISION_TESTS=false
+TEST_PATTERN=""
+
+# Help function
+show_help() {
+    cat << EOF
+Usage: $0 [OPTIONS]
+
+Trigger the integration test recording workflow remotely. This way you do not need to have Ollama running locally.
+
+OPTIONS:
+    -b, --branch BRANCH         Branch to run the workflow on (defaults to current branch)
+    -s, --test-subdirs DIRS     Comma-separated list of test subdirectories to run (REQUIRED)
+    -p, --test-provider PROVIDER Test provider to use: vllm or ollama (default: ollama)
+    -v, --run-vision-tests      Include vision tests in the recording
+    -k, --test-pattern PATTERN  Regex pattern to pass to pytest -k
+    -h, --help                  Show this help message
+
+EXAMPLES:
+    # Record tests for current branch with agents subdirectory
+    $0 --test-subdirs "agents"
+
+    # Record tests for specific branch with vision tests
+    $0 -b my-feature-branch --test-subdirs "inference" --run-vision-tests
+
+    # Record multiple test subdirectories with specific provider
+    $0 --test-subdirs "agents,inference" --test-provider vllm
+
+    # Record tests matching a specific pattern
+    $0 --test-subdirs "inference" --test-pattern "test_streaming"
+
+EOF
+}
+
+# PREREQUISITES:
+#     - GitHub CLI (gh) must be installed and authenticated
+#     - jq must be installed for JSON parsing
+#     - You must be in a git repository that is a fork or clone of llamastack/llama-stack
+#     - The branch must exist on the remote repository where you want to run the workflow
+#     - You must specify test subdirectories to run with -s/--test-subdirs
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -b|--branch)
+            BRANCH="$2"
+            shift 2
+            ;;
+        -s|--test-subdirs)
+            TEST_SUBDIRS="$2"
+            shift 2
+            ;;
+        -p|--test-provider)
+            TEST_PROVIDER="$2"
+            shift 2
+            ;;
+        -v|--run-vision-tests)
+            RUN_VISION_TESTS=true
+            shift
+            ;;
+        -k|--test-pattern)
+            TEST_PATTERN="$2"
+            shift 2
+            ;;
+        -h|--help)
+            show_help
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            show_help
+            exit 1
+            ;;
+    esac
+done
+
+# Validate required parameters
+if [[ -z "$TEST_SUBDIRS" ]]; then
+    echo "Error: --test-subdirs is required"
+    echo "Please specify which test subdirectories to run, e.g.:"
+    echo "  $0 --test-subdirs \"agents,inference\""
+    echo "  $0 --test-subdirs \"inference\" --run-vision-tests"
+    echo ""
+    exit 1
+fi
+
+# Validate test provider
+if [[ "$TEST_PROVIDER" != "vllm" && "$TEST_PROVIDER" != "ollama" ]]; then
+    echo "❌ Error: Invalid test provider '$TEST_PROVIDER'"
+    echo "   Supported providers: vllm, ollama"
+    echo "   Example: $0 --test-subdirs \"agents\" --test-provider vllm"
+    exit 1
+fi
+
+# Check if required tools are installed
+if ! command -v gh &> /dev/null; then
+    echo "Error: GitHub CLI (gh) is not installed. Please install it from https://cli.github.com/"
+    exit 1
+fi
+
+if ! gh auth status &> /dev/null; then
+    echo "Error: GitHub CLI is not authenticated. Please run 'gh auth login'"
+    exit 1
+fi
+
+# If no branch specified, use current branch
+if [[ -z "$BRANCH" ]]; then
+    BRANCH=$(git branch --show-current)
+    echo "No branch specified, using current branch: $BRANCH"
+
+    # Optionally look for associated PR for context (not required)
+    echo "Looking for associated PR..."
+
+    # Search for PRs in the main repo that might match this branch
+    # This searches llamastack/llama-stack for any PR with this head branch name
+    if PR_INFO=$(gh pr list --repo llamastack/llama-stack --head "$BRANCH" --json number,headRefName,headRepository,headRepositoryOwner,url,state --limit 1 2>/dev/null) && [[ "$PR_INFO" != "[]" ]]; then
+        # Parse PR info using jq
+        PR_NUMBER=$(echo "$PR_INFO" | jq -r '.[0].number')
+        PR_HEAD_REPO=$(echo "$PR_INFO" | jq -r '.[0].headRepositoryOwner.login // "llamastack"')
+        PR_URL=$(echo "$PR_INFO" | jq -r '.[0].url')
+        PR_STATE=$(echo "$PR_INFO" | jq -r '.[0].state')
+
+        if [[ -n "$PR_NUMBER" && -n "$PR_HEAD_REPO" ]]; then
+            echo "✅ Found associated PR #$PR_NUMBER ($PR_STATE)"
+            echo "   URL: $PR_URL"
+            echo "   Head repository: $PR_HEAD_REPO/llama-stack"
+
+            # Check PR state and block if merged
+            if [[ "$PR_STATE" == "CLOSED" ]]; then
+                echo "ℹ️  Note: This PR is closed, but workflow can still run to update recordings."
+            elif [[ "$PR_STATE" == "MERGED" ]]; then
+                echo "❌ Error: This PR is already merged."
+                echo "   Cannot record tests for a merged PR since changes can't be committed back."
+                echo "   Create a new branch/PR if you need to record new tests."
+                exit 1
+            fi
+        fi
+    else
+        echo "ℹ️  No associated PR found for branch '$BRANCH'"
+        echo "That's fine - the workflow just needs a pushed branch to run."
+    fi
+    echo ""
+fi
+
+# Determine the target repository for workflow dispatch based on where the branch actually exists
+# We need to find which remote has the branch we want to run the workflow on
+
+echo "Determining target repository for workflow..."
+
+# Check if we have PR info with head repository
+if [[ -n "$PR_HEAD_REPO" ]]; then
+    # Use the repository from the PR head
+    TARGET_REPO="$PR_HEAD_REPO/llama-stack"
+    echo "📍 Using PR head repository: $TARGET_REPO"
+
+    if [[ "$PR_HEAD_REPO" == "llamastack" ]]; then
+        REPO_CONTEXT=""
+    else
+        REPO_CONTEXT="--repo $TARGET_REPO"
+    fi
+else
+    # Fallback: find which remote has the branch
+    BRANCH_REMOTE=""
+    for remote in $(git remote); do
+        if git ls-remote --heads "$remote" "$BRANCH" | grep -q "$BRANCH"; then
+            REMOTE_URL=$(git remote get-url "$remote")
+            if [[ "$REMOTE_URL" == *"/llama-stack"* ]]; then
+                REPO_OWNER=$(echo "$REMOTE_URL" | sed -n 's/.*[:/]\([^/]*\)\/llama-stack.*/\1/p')
+                echo "📍 Found branch '$BRANCH' on remote '$remote' ($REPO_OWNER/llama-stack)"
+                TARGET_REPO="$REPO_OWNER/llama-stack"
+                BRANCH_REMOTE="$remote"
+                break
+            fi
+        fi
+    done
+
+    if [[ -z "$BRANCH_REMOTE" ]]; then
+        echo "Error: Could not find branch '$BRANCH' on any llama-stack remote"
+        echo ""
+        echo "This could mean:"
+        echo "   - The branch doesn't exist on any remote yet (push it first)"
+        echo "   - The branch name is misspelled"
+        echo "   - No llama-stack remotes are configured"
+        echo ""
+        echo "Available remotes:"
+        git remote -v
+        echo ""
+        echo "To push your branch: git push <remote> $BRANCH"
+        echo "Common remotes to try: origin, upstream, your-username"
+        exit 1
+    fi
+
+    if [[ "$TARGET_REPO" == "llamastack/llama-stack" ]]; then
+        REPO_CONTEXT=""
+    else
+        REPO_CONTEXT="--repo $TARGET_REPO"
+    fi
+fi
+
+echo "   Workflow will run on: $TARGET_REPO"
+
+# Verify the target repository has the workflow file
+echo "Verifying workflow exists on target repository..."
+if ! gh api "repos/$TARGET_REPO/contents/.github/workflows/record-integration-tests.yml" &>/dev/null; then
+    echo "Error: The recording workflow does not exist on $TARGET_REPO"
+    echo "This could mean:"
+    echo "   - The fork doesn't have the latest workflow file"
+    echo "   - The workflow file was renamed or moved"
+    echo ""
+    if [[ "$TARGET_REPO" != "llamastack/llama-stack" ]]; then
+        echo "Try syncing your fork with upstream:"
+        echo "   git fetch upstream"
+        echo "   git checkout main"
+        echo "   git merge upstream/main"
+        echo "   git push origin main"
+    fi
+    exit 1
+fi
+
+# Build the workflow dispatch command
+echo "Triggering integration test recording workflow..."
+echo "Branch: $BRANCH"
+echo "Test provider: $TEST_PROVIDER"
+echo "Test subdirs: $TEST_SUBDIRS"
+echo "Run vision tests: $RUN_VISION_TESTS"
+echo "Test pattern: ${TEST_PATTERN:-"(none)"}"
+echo ""
+
+# Prepare inputs for gh workflow run
+INPUTS="-f test-subdirs='$TEST_SUBDIRS'"
+if [[ -n "$TEST_PROVIDER" ]]; then
+    INPUTS="$INPUTS -f test-provider='$TEST_PROVIDER'"
+fi
+if [[ "$RUN_VISION_TESTS" == "true" ]]; then
+    INPUTS="$INPUTS -f run-vision-tests=true"
+fi
+if [[ -n "$TEST_PATTERN" ]]; then
+    INPUTS="$INPUTS -f test-pattern='$TEST_PATTERN'"
+fi
+
+# Run the workflow
+WORKFLOW_CMD="gh workflow run record-integration-tests.yml --ref $BRANCH $REPO_CONTEXT $INPUTS"
+echo "Running: $WORKFLOW_CMD"
+echo ""
+
+if eval "$WORKFLOW_CMD"; then
+    echo "✅ Workflow triggered successfully!"
+    echo ""
+    echo "You can monitor the workflow run at:"
+    echo "https://github.com/$TARGET_REPO/actions/workflows/record-integration-tests.yml"
+    echo ""
+    if [[ -n "$REPO_CONTEXT" ]]; then
+        echo "Or use: gh run list --workflow=record-integration-tests.yml $REPO_CONTEXT"
+        echo "And then: gh run watch <RUN_ID> $REPO_CONTEXT"
+    else
+        echo "Or use: gh run list --workflow=record-integration-tests.yml"
+        echo "And then: gh run watch <RUN_ID>"
+    fi
+else
+    echo "❌ Failed to trigger workflow"
+    exit 1
+fi
diff --git a/tests/README.md b/tests/README.md
index abbfc6d60..3b129fbd9 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -60,7 +60,9 @@ FIREWORKS_API_KEY=your_key pytest -sv tests/integration/inference --stack-config
 
 ### Re-recording tests
 
-If you want to re-record tests, you can do so with:
+#### Local Re-recording (Manual Setup Required)
+
+If you want to re-record tests locally, you can do so with:
 
 ```bash
 LLAMA_STACK_TEST_INFERENCE_MODE=record \
@@ -71,7 +73,6 @@ LLAMA_STACK_TEST_INFERENCE_MODE=record \
 
 This will record new API responses and overwrite the existing recordings.
 
-
 ```{warning}
 
 You must be careful when re-recording. CI workflows assume a specific setup for running the replay-mode tests. You must re-record the tests in the same way as the CI workflows. This means
@@ -79,6 +80,34 @@ You must be careful when re-recording. CI workflows assume a specific setup for
 - you are using the `starter` distribution.
 ```
 
+#### Remote Re-recording (Recommended)
+
+**For easier re-recording without local setup**, use the automated recording workflow:
+
+```bash
+# Record tests for specific test subdirectories
+./scripts/github/schedule-record-workflow.sh --test-subdirs "agents,inference"
+
+# Record with vision tests enabled
+./scripts/github/schedule-record-workflow.sh --test-subdirs "inference" --run-vision-tests
+
+# Record with specific provider
+./scripts/github/schedule-record-workflow.sh --test-subdirs "agents" --test-provider vllm
+```
+
+This script:
+- 🚀 **Runs in GitHub Actions** - no local Ollama setup required
+- 🔍 **Auto-detects your branch** and associated PR
+- 🍴 **Works from forks** - handles repository context automatically
+- ✅ **Commits recordings back** to your branch
+
+**Prerequisites:**
+- GitHub CLI: `brew install gh && gh auth login`
+- jq: `brew install jq`
+- Your branch pushed to a remote
+
+**Supported providers:** `vllm`, `ollama`
+
 
 ### Next Steps
 
diff --git a/tests/integration/README.md b/tests/integration/README.md
index 427b905b4..46d66fd79 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -134,6 +134,15 @@ cat recordings/responses/abc123.json | jq '.'
 ```
 
 ### Re-recording Tests
+
+#### Remote Re-recording (Recommended)
+Use the automated workflow script for easier re-recording:
+```bash
+./scripts/github/schedule-record-workflow.sh --test-subdirs "inference,agents"
+```
+See the [main testing guide](../README.md#remote-re-recording-recommended) for full details.
+
+#### Local Re-recording
 ```bash
 # Re-record specific tests
 LLAMA_STACK_TEST_INFERENCE_MODE=record \